diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java new file mode 100644 index 0000000..2049a34 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + * Enumeration of various states of a RMAppAttempt. + */ +@Public +@Stable +public enum YarnApplicationAttemptState { + /** AppAttempt was just created. */ + NEW, + + /** AppAttempt has been submitted. */ + SUBMITTED, + + /**AppAttempt was scheduled*/ + SCHEDULED, + + /**Acquired AM Container from Scheduler and Saving AppAttempt Data*/ + ALLOCATED_SAVING, + + /**AppAttempt Data was saved*/ + ALLOCATED, + + /**AppAttempt was launched*/ + LAUNCHED, + + /** AppAttempt failed. */ + FAILED, + + /** AppAttempt is currently running. */ + RUNNING, + + /** AppAttempt is waiting for state bing saved*/ + FINAL_SAVING, + + /** AppAttempt is finishing. */ + FINISHING, + + /** AppAttempt finished successfully. */ + FINISHED, + + /** AppAttempt was terminated by a user or admin. */ + KILLED + +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto index 6a8efa0..64e7af8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto @@ -35,6 +35,25 @@ message ApplicationHistoryDataProto { optional int64 finish_time = 8; optional string diagnostics_info = 9; optional FinalApplicationStatusProto final_application_status = 10; + optional YarnApplicationStateProto yarn_application_state = 11; +} + +message ApplicationStartDataProto { + optional ApplicationIdProto application_id = 1; + optional string application_name = 2; + optional string application_type = 3; + optional string user = 4; + optional string queue = 5; + optional int64 submit_time = 6; + optional int64 start_time = 7; +} + +message ApplicationFinishDataProto { + optional ApplicationIdProto application_id = 1; + optional int64 finish_time = 2; + optional string diagnostics_info = 3; + optional FinalApplicationStatusProto final_application_status = 4; + optional YarnApplicationStateProto yarn_application_state = 5; } message ApplicationAttemptHistoryDataProto { @@ -45,6 +64,22 @@ message ApplicationAttemptHistoryDataProto { optional string diagnostics_info = 5; optional FinalApplicationStatusProto final_application_status = 6; optional ContainerIdProto master_container_id = 7; + optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 8; +} + +message ApplicationAttemptStartDataProto { + optional ApplicationAttemptIdProto application_attempt_id = 1; + optional string host = 2; + optional int32 rpc_port = 3; + optional ContainerIdProto master_container_id = 4; +} + +message ApplicationAttemptFinishDataProto { + optional ApplicationAttemptIdProto application_attempt_id = 1; + optional string tracking_url = 2; + optional string diagnostics_info = 3; + optional FinalApplicationStatusProto final_application_status = 4; + optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 5; } message ContainerHistoryDataProto { @@ -56,5 +91,23 @@ message ContainerHistoryDataProto { optional int64 finish_time = 6; optional string diagnostics_info = 7; optional string log_url = 8; - optional ContainerStateProto final_container_status = 9; + optional int32 container_exit_status = 9; + optional ContainerStateProto container_state = 10; +} + +message ContainerStartDataProto { + optional ContainerIdProto container_id = 1; + optional ResourceProto allocated_resource = 2; + optional NodeIdProto assigned_node_id = 3; + optional PriorityProto priority = 4; + optional int64 start_time = 5; +} + +message ContainerFinishDataProto { + optional ContainerIdProto container_id = 1; + optional int64 finish_time = 2; + optional string diagnostics_info = 3; + optional string log_url = 4; + optional int32 container_exit_status = 5; + optional ContainerStateProto container_state = 6; } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 7dc62fc..7834c6f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -98,6 +98,21 @@ enum YarnApplicationStateProto { KILLED = 8; } +enum YarnApplicationAttemptStateProto { + APP_ATTEMPT_NEW = 1; + APP_ATTEMPT_SUBMITTED = 2; + APP_ATTEMPT_SCHEDULED = 3; + APP_ATTEMPT_ALLOCATED_SAVING = 4; + APP_ATTEMPT_ALLOCATED = 5; + APP_ATTEMPT_LAUNCHED = 6; + APP_ATTEMPT_FAILED = 7; + APP_ATTEMPT_RUNNING = 8; + APP_ATTEMPT_FINAL_SAVING = 9; + APP_ATTEMPT_FINISHING = 10; + APP_ATTEMPT_FINISHED = 11; + APP_ATTEMPT_KILLED = 12; +} + enum FinalApplicationStatusProto { APP_UNDEFINED = 0; APP_SUCCEEDED = 1; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index b660f7d..8d73744 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto; @@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import com.google.protobuf.ByteString; @@ -97,6 +99,21 @@ public static YarnApplicationState convertFromProtoFormat(YarnApplicationStatePr } /* + * YarnApplicationAttemptState + */ + private static String YARN_APPLICATION_ATTEMPT_STATE_PREFIX = "APP_ATTEMPT_"; + public static YarnApplicationAttemptStateProto convertToProtoFormat( + YarnApplicationAttemptState e) { + return YarnApplicationAttemptStateProto + .valueOf(YARN_APPLICATION_ATTEMPT_STATE_PREFIX + e.name()); + } + public static YarnApplicationAttemptState convertFromProtoFormat( + YarnApplicationAttemptStateProto e) { + return YarnApplicationAttemptState.valueOf(e.name().replace( + YARN_APPLICATION_ATTEMPT_STATE_PREFIX, "")); + } + + /* * ApplicationResourceUsageReport */ public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java index 3d0db8a..c26faef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java @@ -20,9 +20,18 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.service.Service; +/** + * This class is the abstract of the storage of the application history data. It + * is a {@link Service}, such that the implementation of this class can make use + * of the service life cycle to initialize and cleanup the storage. Users can + * access the storage via {@link ApplicationHistoryReader} and + * {@link ApplicationHistoryWriter} interfaces. + * + */ @InterfaceAudience.Public @InterfaceStability.Unstable -public interface ApplicationHistoryStore extends ApplicationHistoryReader, - ApplicationHistoryWriter { +public interface ApplicationHistoryStore extends Service, + ApplicationHistoryReader, ApplicationHistoryWriter { } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java index c721c63..46e83a1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java @@ -18,11 +18,16 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; +import java.io.IOException; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; -import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; -import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData; /** * It is the interface of writing the application history, exposing the methods @@ -34,25 +39,72 @@ public interface ApplicationHistoryWriter { /** - * This method persists an {@link ApplicationHistoryData} object. - * @param app the {@link ApplicationHistoryData} object - * @throws Throwable + * This method writes the information of RMApp that is available + * when it starts. + * + * @param appStart + * the record of the information of RMApp that is + * available when it starts + * @throws IOException + */ + void applicationStarted(ApplicationStartData appStart) throws IOException; + + /** + * This method writes the information of RMApp that is available + * when it finishes. + * + * @param appFinish + * the record of the information of RMApp that is + * available when it finishes + * @throws IOException + */ + void applicationFinished(ApplicationFinishData appFinish) throws IOException; + + /** + * This method writes the information of RMAppAttempt that is + * available when it starts. + * + * @param appAttemptStart + * the record of the information of RMAppAttempt that is + * available when it starts + * @throws IOException + */ + void applicationAttemptStarted( + ApplicationAttemptStartData appAttemptStart) throws IOException; + + /** + * This method writes the information of RMAppAttempt that is + * available when it finishes. + * + * @param appAttemptFinish + * the record of the information of RMAppAttempt that is + * available when it finishes + * @throws IOException */ - void writeApplication(ApplicationHistoryData app) throws Throwable; + void applicationAttemptFinished( + ApplicationAttemptFinishData appAttemptFinish) throws IOException; /** - * This method persists an {@link ApplicationAttemptHistoryData} object. - * @param appAttempt the {@link ApplicationAttemptHistoryData} object - * @throws Throwable + * This method writes the information of RMContainer that is + * available when it starts. + * + * @param containerStart + * the record of the information of RMContainer that is + * available when it starts + * @throws IOException */ - void writeApplicationAttempt( - ApplicationAttemptHistoryData appAttempt) throws Throwable; + void containerStarted(ContainerStartData containerStart) throws IOException; /** - * This method persists a {@link ContainerHistoryData} object. - * @param container the {@link ContainerHistoryData} object - * @throws Throwable + * This method writes the information of RMContainer that is + * available when it finishes. + * + * @param containerFinish + * the record of the information of RMContainer that is + * available when it finishes + * @throws IOException */ - void writeContainer(ContainerHistoryData container) throws Throwable; + void containerFinished(ContainerFinishData containerFinish) + throws IOException; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java index 0070c35..309ad4b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java @@ -19,46 +19,55 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData; -public class MemoryApplicationHistoryStore implements ApplicationHistoryStore { - - private static MemoryApplicationHistoryStore memStore = null; +/** + * In-memory implementation of {@link ApplicationHistoryStore}. + * This implementation is for test purpose only. If users improperly instantiate + * it, they may encounter reading and writing history data in different memory + * store. + * + */ +@Private +@Unstable +public class MemoryApplicationHistoryStore extends AbstractService + implements ApplicationHistoryStore { - private ConcurrentHashMap applicationData = + private final ConcurrentMap applicationData = new ConcurrentHashMap(); - private ConcurrentHashMap> applicationAttemptData = - new ConcurrentHashMap>(); - private ConcurrentHashMap containerData = - new ConcurrentHashMap(); + private final ConcurrentMap> applicationAttemptData = + new ConcurrentHashMap>(); + private final ConcurrentMap> containerData = + new ConcurrentHashMap>(); - private MemoryApplicationHistoryStore() { - } - - public static MemoryApplicationHistoryStore getMemoryStore() { - if (memStore == null) { - memStore = new MemoryApplicationHistoryStore(); - } - return memStore; + public MemoryApplicationHistoryStore() { + super(MemoryApplicationHistoryStore.class.getName()); } @Override public Map getAllApplications() { - Map listApps = - new HashMap(); - for (ApplicationId appId : applicationData.keySet()) { - listApps.put(appId, applicationData.get(appId)); - } - return listApps; + return new HashMap( + applicationData); } @Override @@ -67,111 +76,209 @@ public ApplicationHistoryData getApplication(ApplicationId appId) { } @Override - public Map getApplicationAttempts( - ApplicationId appId) { - Map listAttempts = - null; - ConcurrentHashMap appAttempts = + public Map + getApplicationAttempts( + ApplicationId appId) { + ConcurrentMap subMap = applicationAttemptData.get(appId); - if (appAttempts != null) { - listAttempts = - new HashMap(); - for (ApplicationAttemptId attemptId : appAttempts.keySet()) { - listAttempts.put(attemptId, appAttempts.get(attemptId)); - } + if (subMap == null) { + return Collections.emptyMap(); + } else { + return new HashMap(subMap); } - return listAttempts; } @Override public ApplicationAttemptHistoryData getApplicationAttempt( ApplicationAttemptId appAttemptId) { - ApplicationAttemptHistoryData appAttemptHistoryData = null; - ConcurrentHashMap appAttempts = + ConcurrentMap subMap = applicationAttemptData.get(appAttemptId.getApplicationId()); - if (appAttempts != null) { - appAttemptHistoryData = appAttempts.get(appAttemptId); + if (subMap == null) { + return null; + } else { + return subMap.get(appAttemptId); } - return appAttemptHistoryData; } @Override - public ContainerHistoryData getAMContainer(ApplicationAttemptId appAttemptId) { - ContainerHistoryData Container = null; - ConcurrentHashMap appAttempts = - applicationAttemptData.get(appAttemptId.getApplicationId()); - if (appAttempts != null) { - containerData.get(appAttempts.get(appAttemptId).getMasterContainerId()); + public ContainerHistoryData getAMContainer( + ApplicationAttemptId appAttemptId) { + ApplicationAttemptHistoryData appAttempt = + getApplicationAttempt(appAttemptId); + if (appAttempt == null || appAttempt.getMasterContainerId() == null) { + return null; + } else { + return getContainer(appAttempt.getMasterContainerId()); } - return Container; } @Override public ContainerHistoryData getContainer(ContainerId containerId) { - return containerData.get(containerId); + Map subMap = + containerData.get(containerId.getApplicationAttemptId()); + if (subMap == null) { + return null; + } else { + return subMap.get(containerId); + } } @Override - public void writeApplication(ApplicationHistoryData app) throws Throwable { - if (app != null) { - ApplicationHistoryData oldData = - applicationData.putIfAbsent(app.getApplicationId(), app); - if (oldData != null) { - throw new IOException("This application " - + app.getApplicationId().toString() + " is already present."); - } + public Map getContainers( + ApplicationAttemptId appAttemptId) throws IOException { + ConcurrentMap subMap = + containerData.get(appAttemptId); + if (subMap == null) { + return Collections.emptyMap(); + } else { + return new HashMap(subMap); } } @Override - public void writeApplicationAttempt(ApplicationAttemptHistoryData appAttempt) - throws Throwable { - if (appAttempt != null) { - if (applicationAttemptData.containsKey(appAttempt - .getApplicationAttemptId().getApplicationId())) { - ConcurrentHashMap appAttemptmap = - applicationAttemptData.get(appAttempt.getApplicationAttemptId() - .getApplicationId()); - ApplicationAttemptHistoryData oldAppAttempt = - appAttemptmap.putIfAbsent(appAttempt.getApplicationAttemptId(), - appAttempt); - if (oldAppAttempt != null) { - throw new IOException("This application attempt " - + appAttempt.getApplicationAttemptId().toString() - + " already present."); - } - } else { - ConcurrentHashMap appAttemptmap = - new ConcurrentHashMap(); - appAttemptmap.put(appAttempt.getApplicationAttemptId(), appAttempt); - applicationAttemptData.putIfAbsent(appAttempt.getApplicationAttemptId() - .getApplicationId(), appAttemptmap); - } + public void applicationStarted(ApplicationStartData appStart) + throws IOException { + ApplicationHistoryData oldData = + applicationData.putIfAbsent(appStart.getApplicationId(), + ApplicationHistoryData.newInstance( + appStart.getApplicationId(), + appStart.getApplicationName(), + appStart.getApplicationType(), + appStart.getQueue(), + appStart.getUser(), + appStart.getSubmitTime(), + appStart.getStartTime(), + Long.MAX_VALUE, null, null, null)); + if (oldData != null) { + throw new IOException("The start information of application " + + appStart.getApplicationId() + " is already stored."); } } @Override - public void writeContainer(ContainerHistoryData container) throws Throwable { - if (container != null) { - ContainerHistoryData oldContainer = - containerData.putIfAbsent(container.getContainerId(), container); - if (oldContainer != null) { - throw new IOException("This container " - + container.getContainerId().toString() + " is already present."); - } + public void applicationFinished(ApplicationFinishData appFinish) + throws IOException { + ApplicationHistoryData data = + applicationData.get(appFinish.getApplicationId()); + if (data == null) { + throw new IOException("The finish information of application " + + appFinish.getApplicationId() + " is stored before the start" + + " information."); } + // Make the assumption that YarnApplicationState should not be null if + // the finish information is already recorded + if (data.getYarnApplicationState() != null) { + throw new IOException("The finish information of application " + + appFinish.getApplicationId() + " is already stored."); + } + data.setFinishTime(appFinish.getFinishTime()); + data.setDiagnosticsInfo(appFinish.getDiagnosticsInfo()); + data.setFinalApplicationStatus(appFinish.getFinalApplicationStatus()); + data.setYarnApplicationState(appFinish.getYarnApplicationState()); } @Override - public Map getContainers( - ApplicationAttemptId appAttemptId) throws IOException { - HashMap containers = - new HashMap(); - for (ContainerId container : containerData.keySet()) { - if (container.getApplicationAttemptId().equals(appAttemptId)) { - containers.put(container, containerData.get(container)); - } + public void applicationAttemptStarted( + ApplicationAttemptStartData appAttemptStart) throws IOException { + ConcurrentMap subMap = + getSubMap(appAttemptStart.getApplicationAttemptId().getApplicationId()); + ApplicationAttemptHistoryData oldData = subMap.putIfAbsent( + appAttemptStart.getApplicationAttemptId(), + ApplicationAttemptHistoryData.newInstance( + appAttemptStart.getApplicationAttemptId(), + appAttemptStart.getHost(), + appAttemptStart.getRPCPort(), + appAttemptStart.getMasterContainerId(), + null, null, null, null)); + if (oldData != null) { + throw new IOException("The start information of application attempt " + + appAttemptStart.getApplicationAttemptId() + + " is already stored."); } - return containers; } + + @Override + public void applicationAttemptFinished( + ApplicationAttemptFinishData appAttemptFinish) throws IOException { + ConcurrentMap subMap = + getSubMap(appAttemptFinish.getApplicationAttemptId().getApplicationId()); + ApplicationAttemptHistoryData data = + subMap.get(appAttemptFinish.getApplicationAttemptId()); + if (data == null) { + throw new IOException("The finish information of application attempt " + + appAttemptFinish.getApplicationAttemptId() + " is stored before" + + " the start information."); + } + // Make the assumption that YarnApplicationAttemptState should not be null + // if the finish information is already recorded + if (data.getYarnApplicationAttemptState() != null) { + throw new IOException("The finish information of application attempt " + + appAttemptFinish.getApplicationAttemptId() + + " is already stored."); + } + data.setTrackingURL(appAttemptFinish.getTrackingURL()); + data.setDiagnosticsInfo(appAttemptFinish.getDiagnosticsInfo()); + data.setFinalApplicationStatus(appAttemptFinish.getFinalApplicationStatus()); + data.setYarnApplicationAttemptState(appAttemptFinish.getYarnApplicationAttemptState()); + } + + private ConcurrentMap + getSubMap(ApplicationId appId) { + applicationAttemptData.putIfAbsent(appId, + new ConcurrentHashMap()); + return applicationAttemptData.get(appId); + } + + @Override + public void containerStarted(ContainerStartData containerStart) + throws IOException { + ConcurrentMap subMap = + getSubMap(containerStart.getContainerId().getApplicationAttemptId()); + ContainerHistoryData oldData = subMap.putIfAbsent( + containerStart.getContainerId(), + ContainerHistoryData.newInstance( + containerStart.getContainerId(), + containerStart.getAllocatedResource(), + containerStart.getAssignedNode(), + containerStart.getPriority(), + containerStart.getStartTime(), + Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null)); + if (oldData != null) { + throw new IOException("The start information of container " + + containerStart.getContainerId() + " is already stored."); + } + } + + @Override + public void containerFinished(ContainerFinishData containerFinish) + throws IOException { + ConcurrentMap subMap = + getSubMap(containerFinish.getContainerId().getApplicationAttemptId()); + ContainerHistoryData data = subMap.get(containerFinish.getContainerId()); + if (data == null) { + throw new IOException("The finish information of container " + + containerFinish.getContainerId() + " is stored before" + + " the start information."); + } + // Make the assumption that ContainerState should not be null if + // the finish information is already recorded + if (data.getContainerState() != null) { + throw new IOException("The finish information of container " + + containerFinish.getContainerId() + " is already stored."); + } + data.setFinishTime(containerFinish.getFinishTime()); + data.setDiagnosticsInfo(containerFinish.getDiagnosticsInfo()); + data.setLogURL(containerFinish.getLogURL()); + data.setContainerExitStatus(containerFinish.getContainerExitStatus()); + data.setContainerState(containerFinish.getContainerState()); + } + + private ConcurrentMap getSubMap( + ApplicationAttemptId appAttemptId) { + containerData.putIfAbsent(appAttemptId, + new ConcurrentHashMap()); + return containerData.get(appAttemptId); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java new file mode 100644 index 0000000..7ba51af --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.util.Records; + +/** + * The class contains the fields that can be determined when + * RMAppAttempt finishes, and that need to be stored persistently. + */ +@Public +@Unstable +public abstract class ApplicationAttemptFinishData { + + @Public + @Unstable + public static ApplicationAttemptFinishData newInstance( + ApplicationAttemptId appAttemptId, String diagnosticsInfo, + String trackingURL, FinalApplicationStatus finalApplicationStatus, + YarnApplicationAttemptState yarnApplicationAttemptState) { + ApplicationAttemptFinishData appAttemptFD = + Records.newRecord(ApplicationAttemptFinishData.class); + appAttemptFD.setApplicationAttemptId(appAttemptId); + appAttemptFD.setDiagnosticsInfo(diagnosticsInfo); + appAttemptFD.setTrackingURL(trackingURL); + appAttemptFD.setFinalApplicationStatus(finalApplicationStatus); + appAttemptFD.setYarnApplicationAttemptState(yarnApplicationAttemptState); + return appAttemptFD; + } + + @Public + @Unstable + public abstract ApplicationAttemptId getApplicationAttemptId(); + + @Public + @Unstable + public abstract void setApplicationAttemptId( + ApplicationAttemptId applicationAttemptId); + + @Public + @Unstable + public abstract String getTrackingURL(); + + @Public + @Unstable + public abstract void setTrackingURL(String trackingURL); + + @Public + @Unstable + public abstract String getDiagnosticsInfo(); + + @Public + @Unstable + public abstract void setDiagnosticsInfo(String diagnosticsInfo); + + @Public + @Unstable + public abstract FinalApplicationStatus getFinalApplicationStatus(); + + @Public + @Unstable + public abstract void setFinalApplicationStatus( + FinalApplicationStatus finalApplicationStatus); + + @Public + @Unstable + public abstract YarnApplicationAttemptState getYarnApplicationAttemptState(); + + @Public + @Unstable + public abstract void setYarnApplicationAttemptState( + YarnApplicationAttemptState yarnApplicationAttemptState); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java index 0c3afa3..deffceb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java @@ -23,69 +23,150 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; /** - * The class contains all the fields that need to be stored persistently for + * The class contains all the fields that are stored persistently for * RMAppAttempt. */ @Public @Unstable -public interface ApplicationAttemptHistoryData { +public class ApplicationAttemptHistoryData { + + private ApplicationAttemptId applicationAttemptId; + + private String host; + + private int rpcPort; + + private String trackingURL; + + private String diagnosticsInfo; + + private FinalApplicationStatus finalApplicationStatus; + + private ContainerId masterContainerId; + + private YarnApplicationAttemptState yarnApplicationAttemptState; + + @Public + @Unstable + public static ApplicationAttemptHistoryData newInstance( + ApplicationAttemptId appAttemptId, String host, int rpcPort, + ContainerId masterContainerId, String diagnosticsInfo, + String trackingURL, + FinalApplicationStatus finalApplicationStatus, + YarnApplicationAttemptState yarnApplicationAttemptState) { + ApplicationAttemptHistoryData appAttemptHD = + new ApplicationAttemptHistoryData(); + appAttemptHD.setApplicationAttemptId(appAttemptId); + appAttemptHD.setHost(host); + appAttemptHD.setRPCPort(rpcPort); + appAttemptHD.setMasterContainerId(masterContainerId); + appAttemptHD.setDiagnosticsInfo(diagnosticsInfo); + appAttemptHD.setTrackingURL(trackingURL); + appAttemptHD.setFinalApplicationStatus(finalApplicationStatus); + appAttemptHD.setYarnApplicationAttemptState(yarnApplicationAttemptState); + return appAttemptHD; + } + + @Public + @Unstable + public ApplicationAttemptId getApplicationAttemptId() { + return applicationAttemptId; + } + + @Public + @Unstable + public void setApplicationAttemptId( + ApplicationAttemptId applicationAttemptId) { + this.applicationAttemptId = applicationAttemptId; + } @Public @Unstable - ApplicationAttemptId getApplicationAttemptId(); + public String getHost() { + return host; + } @Public @Unstable - void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId); + public void setHost(String host) { + this.host = host; + } @Public @Unstable - String getHost(); + public int getRPCPort() { + return rpcPort; + } @Public @Unstable - void setHost(String host); + public void setRPCPort(int rpcPort) { + this.rpcPort = rpcPort; + } @Public @Unstable - int getRPCPort(); + public String getTrackingURL() { + return trackingURL; + } @Public @Unstable - void setRPCPort(int rpcPort); + public void setTrackingURL(String trackingURL) { + this.trackingURL = trackingURL; + } @Public @Unstable - String getTrackingURL(); + public String getDiagnosticsInfo() { + return diagnosticsInfo; + } @Public @Unstable - void setTrackingURL(String trackingURL); + public void setDiagnosticsInfo(String diagnosticsInfo) { + this.diagnosticsInfo = diagnosticsInfo; + } @Public @Unstable - String getDiagnosticsInfo(); + public FinalApplicationStatus getFinalApplicationStatus() { + return finalApplicationStatus; + } @Public @Unstable - void setDiagnosticsInfo(String diagnosticsInfo); + public void setFinalApplicationStatus( + FinalApplicationStatus finalApplicationStatus) { + this.finalApplicationStatus = finalApplicationStatus; + } @Public @Unstable - FinalApplicationStatus getFinalApplicationStatus(); + public ContainerId getMasterContainerId() { + return masterContainerId; + } @Public @Unstable - void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus); + public void setMasterContainerId(ContainerId masterContainerId) { + this.masterContainerId = masterContainerId; + } @Public @Unstable - ContainerId getMasterContainerId(); + public YarnApplicationAttemptState getYarnApplicationAttemptState() { + return yarnApplicationAttemptState; + } @Public @Unstable - void setMasterContainerId(ContainerId masterContainerId); + public void setYarnApplicationAttemptState( + YarnApplicationAttemptState yarnApplicationAttemptState) { + this.yarnApplicationAttemptState = yarnApplicationAttemptState; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java new file mode 100644 index 0000000..7ca43fa --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.util.Records; + +/** + * The class contains the fields that can be determined when + * RMAppAttempt starts, and that need to be stored persistently. + */ +@Public +@Unstable +public abstract class ApplicationAttemptStartData { + + @Public + @Unstable + public static ApplicationAttemptStartData newInstance( + ApplicationAttemptId appAttemptId, String host, int rpcPort, + ContainerId masterContainerId) { + ApplicationAttemptStartData appAttemptSD = + Records.newRecord(ApplicationAttemptStartData.class); + appAttemptSD.setApplicationAttemptId(appAttemptId); + appAttemptSD.setHost(host); + appAttemptSD.setRPCPort(rpcPort); + appAttemptSD.setMasterContainerId(masterContainerId); + return appAttemptSD; + } + + @Public + @Unstable + public abstract ApplicationAttemptId getApplicationAttemptId(); + + @Public + @Unstable + public abstract void setApplicationAttemptId( + ApplicationAttemptId applicationAttemptId); + + @Public + @Unstable + public abstract String getHost(); + + @Public + @Unstable + public abstract void setHost(String host); + + @Public + @Unstable + public abstract int getRPCPort(); + + @Public + @Unstable + public abstract void setRPCPort(int rpcPort); + + @Public + @Unstable + public abstract ContainerId getMasterContainerId(); + + @Public + @Unstable + public abstract void setMasterContainerId(ContainerId masterContainerId); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java new file mode 100644 index 0000000..e94bfdd --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.util.Records; + +/** + * The class contains the fields that can be determined when + * RMApp finishes, and that need to be stored persistently. + */ +@Public +@Unstable +public abstract class ApplicationFinishData { + + @Public + @Unstable + public static ApplicationFinishData newInstance(ApplicationId applicationId, + long finishTime, String diagnosticsInfo, + FinalApplicationStatus finalApplicationStatus, + YarnApplicationState yarnApplicationState) { + ApplicationFinishData appFD = + Records.newRecord(ApplicationFinishData.class); + appFD.setApplicationId(applicationId); + appFD.setFinishTime(finishTime); + appFD.setDiagnosticsInfo(diagnosticsInfo); + appFD.setFinalApplicationStatus(finalApplicationStatus); + appFD.setYarnApplicationState(yarnApplicationState); + return appFD; + } + + @Public + @Unstable + public abstract ApplicationId getApplicationId(); + + @Public + @Unstable + public abstract void setApplicationId(ApplicationId applicationId); + + @Public + @Unstable + public abstract long getFinishTime(); + + @Public + @Unstable + public abstract void setFinishTime(long finishTime); + + @Public + @Unstable + public abstract String getDiagnosticsInfo(); + + @Public + @Unstable + public abstract void setDiagnosticsInfo(String diagnosticsInfo); + + @Public + @Unstable + public abstract FinalApplicationStatus getFinalApplicationStatus(); + + @Public + @Unstable + public abstract void setFinalApplicationStatus( + FinalApplicationStatus finalApplicationStatus); + + @Public + @Unstable + public abstract YarnApplicationState getYarnApplicationState(); + + @Public + @Unstable + public abstract void setYarnApplicationState( + YarnApplicationState yarnApplicationState); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java index 1512864..905b691 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java @@ -22,93 +22,192 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; /** - * The class contains all the fields that need to be stored persistently for + * The class contains all the fields that are stored persistently for * RMApp. */ @Public @Unstable -public interface ApplicationHistoryData { +public class ApplicationHistoryData { + + private ApplicationId applicationId; + + private String applicationName; + + private String applicationType; + + private String user; + + private String queue; + + private long submitTime; + + private long startTime; + + private long finishTime; + + private String diagnosticsInfo; + + private FinalApplicationStatus finalApplicationStatus; + + private YarnApplicationState yarnApplicationState; + + @Public + @Unstable + public static ApplicationHistoryData newInstance(ApplicationId applicationId, + String applicationName, String applicationType, String queue, + String user, long submitTime, long startTime, long finishTime, + String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus, + YarnApplicationState yarnApplicationState) { + ApplicationHistoryData appHD = new ApplicationHistoryData(); + appHD.setApplicationId(applicationId); + appHD.setApplicationName(applicationName); + appHD.setApplicationType(applicationType); + appHD.setQueue(queue); + appHD.setUser(user); + appHD.setSubmitTime(submitTime); + appHD.setStartTime(startTime); + appHD.setFinishTime(finishTime); + appHD.setDiagnosticsInfo(diagnosticsInfo); + appHD.setFinalApplicationStatus(finalApplicationStatus); + appHD.setYarnApplicationState(yarnApplicationState); + return appHD; + } + + @Public + @Unstable + public ApplicationId getApplicationId() { + return applicationId; + } + + @Public + @Unstable + public void setApplicationId(ApplicationId applicationId) { + this.applicationId = applicationId; + } @Public @Unstable - ApplicationId getApplicationId(); + public String getApplicationName() { + return applicationName; + } @Public @Unstable - void setApplicationId(ApplicationId applicationId); + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } @Public @Unstable - String getApplicationName(); + public String getApplicationType() { + return applicationType; + } @Public @Unstable - void setApplicationName(String applicationName); + public void setApplicationType(String applicationType) { + this.applicationType = applicationType; + } @Public @Unstable - String getApplicationType(); + public String getUser() { + return user; + } @Public @Unstable - void setApplicationType(String applicationType); + public void setUser(String user) { + this.user = user; + } @Public @Unstable - String getUser(); + public String getQueue() { + return queue; + } @Public @Unstable - void setUser(String user); + public void setQueue(String queue) { + this.queue = queue; + } @Public @Unstable - String getQueue(); + public long getSubmitTime() { + return submitTime; + } @Public @Unstable - void setQueue(String queue); + public void setSubmitTime(long submitTime) { + this.submitTime = submitTime; + } @Public @Unstable - long getSubmitTime(); + public long getStartTime() { + return startTime; + } @Public @Unstable - void setSubmitTime(long submitTime); + public void setStartTime(long startTime) { + this.startTime = startTime; + } @Public @Unstable - long getStartTime(); + public long getFinishTime() { + return finishTime; + } @Public @Unstable - void setStartTime(long startTime); + public void setFinishTime(long finishTime) { + this.finishTime = finishTime; + } @Public @Unstable - long getFinishTime(); + public String getDiagnosticsInfo() { + return diagnosticsInfo; + } @Public @Unstable - void setFinishTime(long finishTime); + public void setDiagnosticsInfo(String diagnosticsInfo) { + this.diagnosticsInfo = diagnosticsInfo; + } @Public @Unstable - String getDiagnosticsInfo(); + public FinalApplicationStatus getFinalApplicationStatus() { + return finalApplicationStatus; + } @Public @Unstable - void setDiagnosticsInfo(String diagnosticInfo); + public void setFinalApplicationStatus( + FinalApplicationStatus finalApplicationStatus) { + this.finalApplicationStatus = finalApplicationStatus; + } @Public @Unstable - FinalApplicationStatus getFinalApplicationStatus(); + public YarnApplicationState getYarnApplicationState() { + return this.yarnApplicationState; + } @Public @Unstable - void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus); + public void setYarnApplicationState( + YarnApplicationState yarnApplicationState) { + this.yarnApplicationState = yarnApplicationState; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java new file mode 100644 index 0000000..021a061 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.Records; + +/** + * The class contains the fields that can be determined when + * RMApp starts, and that need to be stored persistently. + */ +@Public +@Unstable +public abstract class ApplicationStartData { + + @Public + @Unstable + public static ApplicationStartData newInstance( + ApplicationId applicationId, String applicationName, + String applicationType, String queue, String user, + long submitTime, long startTime) { + ApplicationStartData appSD = + Records.newRecord(ApplicationStartData.class); + appSD.setApplicationId(applicationId); + appSD.setApplicationName(applicationName); + appSD.setApplicationType(applicationType); + appSD.setQueue(queue); + appSD.setUser(user); + appSD.setSubmitTime(submitTime); + appSD.setStartTime(startTime); + return appSD; + } + + @Public + @Unstable + public abstract ApplicationId getApplicationId(); + + @Public + @Unstable + public abstract void setApplicationId(ApplicationId applicationId); + + @Public + @Unstable + public abstract String getApplicationName(); + + @Public + @Unstable + public abstract void setApplicationName(String applicationName); + + @Public + @Unstable + public abstract String getApplicationType(); + + @Public + @Unstable + public abstract void setApplicationType(String applicationType); + + @Public + @Unstable + public abstract String getUser(); + + @Public + @Unstable + public abstract void setUser(String user); + + @Public + @Unstable + public abstract String getQueue(); + + @Public + @Unstable + public abstract void setQueue(String queue); + + @Public + @Unstable + public abstract long getSubmitTime(); + + @Public + @Unstable + public abstract void setSubmitTime(long submitTime); + + @Public + @Unstable + public abstract long getStartTime(); + + @Public + @Unstable + public abstract void setStartTime(long startTime); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java new file mode 100644 index 0000000..1c449e3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.util.Records; + +/** + * The class contains the fields that can be determined when + * RMContainer finishes, and that need to be stored persistently. + */ +@Public +@Unstable +public abstract class ContainerFinishData { + + @Public + @Unstable + public static ContainerFinishData newInstance(ContainerId containerId, + long finishTime, String diagnosticsInfo, String logURL, + int containerExitCode, ContainerState containerState) { + ContainerFinishData containerFD = + Records.newRecord(ContainerFinishData.class); + containerFD.setContainerId(containerId); + containerFD.setFinishTime(finishTime); + containerFD.setDiagnosticsInfo(diagnosticsInfo); + containerFD.setLogURL(logURL); + containerFD.setContainerExitStatus(containerExitCode); + containerFD.setContainerState(containerState); + return containerFD; + } + + @Public + @Unstable + public abstract ContainerId getContainerId(); + + @Public + @Unstable + public abstract void setContainerId(ContainerId containerId); + + @Public + @Unstable + public abstract long getFinishTime(); + + @Public + @Unstable + public abstract void setFinishTime(long finishTime); + + @Public + @Unstable + public abstract String getDiagnosticsInfo(); + + @Public + @Unstable + public abstract void setDiagnosticsInfo(String diagnosticsInfo); + + @Public + @Unstable + public abstract String getLogURL(); + + @Public + @Unstable + public abstract void setLogURL(String logURL); + + @Public + @Unstable + public abstract int getContainerExitStatus(); + + @Public + @Unstable + public abstract void setContainerExitStatus(int containerExitStatus); + + @Public + @Unstable + public abstract ContainerState getContainerState(); + + @Public + @Unstable + public abstract void setContainerState(ContainerState containerState); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java index f069b5c..f7660b2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java @@ -27,83 +27,171 @@ import org.apache.hadoop.yarn.api.records.Resource; /** - * The class contains all the fields that need to be stored persistently for + * The class contains all the fields that are stored persistently for * RMContainer. */ @Public @Unstable -public interface ContainerHistoryData { +public class ContainerHistoryData { + + private ContainerId containerId; + + private Resource allocatedResource; + + private NodeId assignedNode; + + private Priority priority; + + private long startTime; + + private long finishTime; + + private String diagnosticsInfo; + + private String logURL; + + private int containerExitStatus; + + private ContainerState containerState; + + @Public + @Unstable + public static ContainerHistoryData newInstance(ContainerId containerId, + Resource allocatedResource, NodeId assignedNode, Priority priority, + long startTime, long finishTime, String diagnosticsInfo, String logURL, + int containerExitCode, ContainerState containerState) { + ContainerHistoryData containerHD = new ContainerHistoryData(); + containerHD.setContainerId(containerId); + containerHD.setAllocatedResource(allocatedResource); + containerHD.setAssignedNode(assignedNode); + containerHD.setPriority(priority); + containerHD.setStartTime(startTime); + containerHD.setFinishTime(finishTime); + containerHD.setDiagnosticsInfo(diagnosticsInfo); + containerHD.setLogURL(logURL); + containerHD.setContainerExitStatus(containerExitCode); + containerHD.setContainerState(containerState); + return containerHD; + } + + @Public + @Unstable + public ContainerId getContainerId() { + return containerId; + } + + @Public + @Unstable + public void setContainerId(ContainerId containerId) { + this.containerId = containerId; + } @Public @Unstable - ContainerId getContainerId(); + public Resource getAllocatedResource() { + return allocatedResource; + } @Public @Unstable - void setContainerId(ContainerId containerId); + public void setAllocatedResource(Resource resource) { + this.allocatedResource = resource; + } @Public @Unstable - Resource getAllocatedResource(); + public NodeId getAssignedNode() { + return assignedNode; + } @Public @Unstable - void setAllocatedResource(Resource resource); + public void setAssignedNode(NodeId nodeId) { + this.assignedNode = nodeId; + } @Public @Unstable - NodeId getAssignedNode(); + public Priority getPriority() { + return priority; + } @Public @Unstable - void setAssignedNode(NodeId nodeId); + public void setPriority(Priority priority) { + this.priority = priority; + } @Public @Unstable - Priority getPriority(); + public long getStartTime() { + return startTime; + } @Public @Unstable - void setPriority(Priority priority); + public void setStartTime(long startTime) { + this.startTime = startTime; + } @Public @Unstable - long getStartTime(); + public long getFinishTime() { + return finishTime; + } @Public @Unstable - void setStartTime(long startTime); + public void setFinishTime(long finishTime) { + this.finishTime = finishTime; + } @Public @Unstable - long getFinishTime(); + public String getDiagnosticsInfo() { + return diagnosticsInfo; + } @Public @Unstable - void setFinishTime(long finishTime); + public void setDiagnosticsInfo(String diagnosticsInfo) { + this.diagnosticsInfo = diagnosticsInfo; + } @Public @Unstable - String getDiagnosticsInfo(); + public String getLogURL() { + return logURL; + } @Public @Unstable - void setDiagnosticsInfo(String diagnosticInfo); + public void setLogURL(String logURL) { + this.logURL = logURL; + } @Public @Unstable - String getLogURL(); + public int getContainerExitStatus() { + return containerExitStatus; + } @Public @Unstable - void setLogURL(String logURL); + public void setContainerExitStatus(int containerExitStatus) { + this.containerExitStatus = containerExitStatus; + } @Public @Unstable - ContainerState getFinalContainerStatus(); + public ContainerState getContainerState() { + return containerState; + } @Public @Unstable - void setFinalContainerStatus(ContainerState finalContainerState); + public void setContainerState(ContainerState containerState) { + this.containerState = containerState; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java new file mode 100644 index 0000000..0c6dd81 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.Records; + +/** + * The class contains the fields that can be determined when + * RMContainer starts, and that need to be stored persistently. + */ +@Public +@Unstable +public abstract class ContainerStartData { + + @Public + @Unstable + public static ContainerStartData newInstance(ContainerId containerId, + Resource allocatedResource, NodeId assignedNode, Priority priority, + long startTime) { + ContainerStartData containerSD = + Records.newRecord(ContainerStartData.class); + containerSD.setContainerId(containerId); + containerSD.setAllocatedResource(allocatedResource); + containerSD.setAssignedNode(assignedNode); + containerSD.setPriority(priority); + containerSD.setStartTime(startTime); + return containerSD; + } + + @Public + @Unstable + public abstract ContainerId getContainerId(); + + @Public + @Unstable + public abstract void setContainerId(ContainerId containerId); + + @Public + @Unstable + public abstract Resource getAllocatedResource(); + + @Public + @Unstable + public abstract void setAllocatedResource(Resource resource); + + @Public + @Unstable + public abstract NodeId getAssignedNode(); + + @Public + @Unstable + public abstract void setAssignedNode(NodeId nodeId); + + @Public + @Unstable + public abstract Priority getPriority(); + + @Public + @Unstable + public abstract void setPriority(Priority priority); + + @Public + @Unstable + public abstract long getStartTime(); + + @Public + @Unstable + public abstract void setStartTime(long startTime); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java new file mode 100644 index 0000000..7ae767b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptFinishDataProto; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptFinishDataProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; +import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData; + +import com.google.protobuf.TextFormat; + +public class ApplicationAttemptFinishDataPBImpl + extends ApplicationAttemptFinishData { + + ApplicationAttemptFinishDataProto proto = + ApplicationAttemptFinishDataProto.getDefaultInstance(); + ApplicationAttemptFinishDataProto.Builder builder = null; + boolean viaProto = false; + + public ApplicationAttemptFinishDataPBImpl() { + builder = ApplicationAttemptFinishDataProto.newBuilder(); + } + + public ApplicationAttemptFinishDataPBImpl( + ApplicationAttemptFinishDataProto proto) { + this.proto = proto; + viaProto = true; + } + + private ApplicationAttemptId applicationAttemptId; + + @Override + public ApplicationAttemptId getApplicationAttemptId() { + if (this.applicationAttemptId != null) { + return this.applicationAttemptId; + } + ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationAttemptId()) { + return null; + } + this.applicationAttemptId = + convertFromProtoFormat(p.getApplicationAttemptId()); + return this.applicationAttemptId; + } + + @Override + public void + setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) { + maybeInitBuilder(); + if (applicationAttemptId == null) { + builder.clearApplicationAttemptId(); + } + this.applicationAttemptId = applicationAttemptId; + } + + @Override + public String getTrackingURL() { + ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasTrackingUrl()) { + return null; + } + return p.getTrackingUrl(); + } + + @Override + public void setTrackingURL(String trackingURL) { + maybeInitBuilder(); + if (trackingURL == null) { + builder.clearTrackingUrl(); + return; + } + builder.setTrackingUrl(trackingURL); + } + + @Override + public String getDiagnosticsInfo() { + ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasDiagnosticsInfo()) { + return null; + } + return p.getDiagnosticsInfo(); + } + + @Override + public void setDiagnosticsInfo(String diagnosticsInfo) { + maybeInitBuilder(); + if (diagnosticsInfo == null) { + builder.clearDiagnosticsInfo(); + return; + } + builder.setDiagnosticsInfo(diagnosticsInfo); + } + + @Override + public FinalApplicationStatus getFinalApplicationStatus() { + ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasFinalApplicationStatus()) { + return null; + } + return convertFromProtoFormat(p.getFinalApplicationStatus()); + } + + @Override + public void setFinalApplicationStatus( + FinalApplicationStatus finalApplicationStatus) { + maybeInitBuilder(); + if (finalApplicationStatus == null) { + builder.clearFinalApplicationStatus(); + return; + } + builder.setFinalApplicationStatus( + convertToProtoFormat(finalApplicationStatus)); + } + + @Override + public YarnApplicationAttemptState getYarnApplicationAttemptState() { + ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasYarnApplicationAttemptState()) { + return null; + } + return convertFromProtoFormat(p.getYarnApplicationAttemptState()); + } + + @Override + public void setYarnApplicationAttemptState(YarnApplicationAttemptState state) { + maybeInitBuilder(); + if (state == null) { + builder.clearYarnApplicationAttemptState(); + return; + } + builder.setYarnApplicationAttemptState(convertToProtoFormat(state)); + } + + public ApplicationAttemptFinishDataProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.applicationAttemptId != null && !((ApplicationAttemptIdPBImpl) + this.applicationAttemptId).getProto().equals( + builder.getApplicationAttemptId())) { + builder.setApplicationAttemptId( + convertToProtoFormat(this.applicationAttemptId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationAttemptFinishDataProto.newBuilder(proto); + } + viaProto = false; + } + + private ApplicationAttemptIdPBImpl convertFromProtoFormat( + ApplicationAttemptIdProto applicationAttemptId) { + return new ApplicationAttemptIdPBImpl(applicationAttemptId); + } + + private ApplicationAttemptIdProto convertToProtoFormat( + ApplicationAttemptId applicationAttemptId) { + return ((ApplicationAttemptIdPBImpl) applicationAttemptId).getProto(); + } + + private FinalApplicationStatus convertFromProtoFormat( + FinalApplicationStatusProto finalApplicationStatus) { + return ProtoUtils.convertFromProtoFormat(finalApplicationStatus); + } + + private FinalApplicationStatusProto convertToProtoFormat( + FinalApplicationStatus finalApplicationStatus) { + return ProtoUtils.convertToProtoFormat(finalApplicationStatus); + } + + private YarnApplicationAttemptStateProto convertToProtoFormat( + YarnApplicationAttemptState state) { + return ProtoUtils.convertToProtoFormat(state); + } + + private YarnApplicationAttemptState convertFromProtoFormat( + YarnApplicationAttemptStateProto yarnApplicationAttemptState) { + return ProtoUtils.convertFromProtoFormat(yarnApplicationAttemptState); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java deleted file mode 100644 index 3362875..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java +++ /dev/null @@ -1,284 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb; - -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; -import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptHistoryDataProto; -import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptHistoryDataProtoOrBuilder; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; -import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; - - -public class ApplicationAttemptHistoryDataPBImpl - extends ProtoBase - implements ApplicationAttemptHistoryData { - - ApplicationAttemptHistoryDataProto proto = - ApplicationAttemptHistoryDataProto.getDefaultInstance(); - ApplicationAttemptHistoryDataProto.Builder builder = null; - boolean viaProto = false; - - public ApplicationAttemptHistoryDataPBImpl() { - builder = ApplicationAttemptHistoryDataProto.newBuilder(); - } - - public ApplicationAttemptHistoryDataPBImpl( - ApplicationAttemptHistoryDataProto proto) { - this.proto = proto; - viaProto = true; - } - - private ApplicationAttemptId applicationAttemptId; - private ContainerId masterContainerId; - - @Override - public ApplicationAttemptId getApplicationAttemptId() { - if (this.applicationAttemptId != null) { - return this.applicationAttemptId; - } - ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasApplicationAttemptId()) { - return null; - } - this.applicationAttemptId = - convertFromProtoFormat(p.getApplicationAttemptId()); - return this.applicationAttemptId; - } - - @Override - public void - setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) { - maybeInitBuilder(); - if (applicationAttemptId == null) { - builder.clearApplicationAttemptId(); - } - this.applicationAttemptId = applicationAttemptId; - } - - @Override - public String getHost() { - ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasHost()) { - return null; - } - return p.getHost(); - } - - @Override - public void setHost(String host) { - maybeInitBuilder(); - if (host == null) { - builder.clearHost(); - return; - } - builder.setHost(host); - } - - @Override - public int getRPCPort() { - ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getRpcPort(); - } - - @Override - public void setRPCPort(int rpcPort) { - maybeInitBuilder(); - builder.setRpcPort(rpcPort); - } - - @Override - public String getTrackingURL() { - ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasTrackingUrl()) { - return null; - } - return p.getTrackingUrl(); - } - - @Override - public void setTrackingURL(String trackingURL) { - maybeInitBuilder(); - if (trackingURL == null) { - builder.clearTrackingUrl(); - return; - } - builder.setTrackingUrl(trackingURL); - } - - @Override - public String getDiagnosticsInfo() { - ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasDiagnosticsInfo()) { - return null; - } - return p.getDiagnosticsInfo(); - } - - @Override - public void setDiagnosticsInfo(String diagnosticsInfo) { - maybeInitBuilder(); - if (diagnosticsInfo == null) { - builder.clearDiagnosticsInfo(); - return; - } - builder.setDiagnosticsInfo(diagnosticsInfo); - } - - @Override - public FinalApplicationStatus getFinalApplicationStatus() { - ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasFinalApplicationStatus()) { - return null; - } - return convertFromProtoFormat(p.getFinalApplicationStatus()); - } - - @Override - public void setFinalApplicationStatus( - FinalApplicationStatus finalApplicationStatus) { - maybeInitBuilder(); - if (finalApplicationStatus == null) { - builder.clearFinalApplicationStatus(); - return; - } - builder.setFinalApplicationStatus( - convertToProtoFormat(finalApplicationStatus)); - } - - @Override - public ContainerId getMasterContainerId() { - if (this.masterContainerId != null) { - return this.masterContainerId; - } - ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasApplicationAttemptId()) { - return null; - } - this.masterContainerId = - convertFromProtoFormat(p.getMasterContainerId()); - return this.masterContainerId; - } - - @Override - public void setMasterContainerId(ContainerId masterContainerId) { - maybeInitBuilder(); - if (masterContainerId == null) { - builder.clearMasterContainerId(); - } - this.masterContainerId = masterContainerId; - } - - @Override - public ApplicationAttemptHistoryDataProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private void mergeLocalToBuilder() { - if (this.applicationAttemptId != null && !((ApplicationAttemptIdPBImpl) - this.applicationAttemptId).getProto().equals( - builder.getApplicationAttemptId())) { - builder.setApplicationAttemptId( - convertToProtoFormat(this.applicationAttemptId)); - } - if (this.masterContainerId != null && !((ContainerIdPBImpl) - this.masterContainerId).getProto().equals( - builder.getMasterContainerId())) { - builder.setMasterContainerId( - convertToProtoFormat(this.masterContainerId)); - } - } - - private void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = ApplicationAttemptHistoryDataProto.newBuilder(proto); - } - viaProto = false; - } - - private ApplicationAttemptIdPBImpl convertFromProtoFormat( - ApplicationAttemptIdProto applicationAttemptId) { - return new ApplicationAttemptIdPBImpl(applicationAttemptId); - } - - private ApplicationAttemptIdProto convertToProtoFormat( - ApplicationAttemptId applicationAttemptId) { - return ((ApplicationAttemptIdPBImpl) applicationAttemptId).getProto(); - } - - private FinalApplicationStatus convertFromProtoFormat( - FinalApplicationStatusProto finalApplicationStatus) { - return ProtoUtils.convertFromProtoFormat(finalApplicationStatus); - } - - private FinalApplicationStatusProto convertToProtoFormat( - FinalApplicationStatus finalApplicationStatus) { - return ProtoUtils.convertToProtoFormat(finalApplicationStatus); - } - - private ContainerIdPBImpl convertFromProtoFormat( - ContainerIdProto containerId) { - return new ContainerIdPBImpl(containerId); - } - - private ContainerIdProto convertToProtoFormat( - ContainerId masterContainerId) { - return ((ContainerIdPBImpl) masterContainerId).getProto(); - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java new file mode 100644 index 0000000..5dbbecc --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptStartDataProto; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptStartDataProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData; + +import com.google.protobuf.TextFormat; + + +public class ApplicationAttemptStartDataPBImpl + extends ApplicationAttemptStartData { + + ApplicationAttemptStartDataProto proto = + ApplicationAttemptStartDataProto.getDefaultInstance(); + ApplicationAttemptStartDataProto.Builder builder = null; + boolean viaProto = false; + + public ApplicationAttemptStartDataPBImpl() { + builder = ApplicationAttemptStartDataProto.newBuilder(); + } + + public ApplicationAttemptStartDataPBImpl( + ApplicationAttemptStartDataProto proto) { + this.proto = proto; + viaProto = true; + } + + private ApplicationAttemptId applicationAttemptId; + private ContainerId masterContainerId; + + @Override + public ApplicationAttemptId getApplicationAttemptId() { + if (this.applicationAttemptId != null) { + return this.applicationAttemptId; + } + ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationAttemptId()) { + return null; + } + this.applicationAttemptId = + convertFromProtoFormat(p.getApplicationAttemptId()); + return this.applicationAttemptId; + } + + @Override + public void + setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) { + maybeInitBuilder(); + if (applicationAttemptId == null) { + builder.clearApplicationAttemptId(); + } + this.applicationAttemptId = applicationAttemptId; + } + + @Override + public String getHost() { + ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasHost()) { + return null; + } + return p.getHost(); + } + + @Override + public void setHost(String host) { + maybeInitBuilder(); + if (host == null) { + builder.clearHost(); + return; + } + builder.setHost(host); + } + + @Override + public int getRPCPort() { + ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getRpcPort(); + } + + @Override + public void setRPCPort(int rpcPort) { + maybeInitBuilder(); + builder.setRpcPort(rpcPort); + } + + @Override + public ContainerId getMasterContainerId() { + if (this.masterContainerId != null) { + return this.masterContainerId; + } + ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationAttemptId()) { + return null; + } + this.masterContainerId = + convertFromProtoFormat(p.getMasterContainerId()); + return this.masterContainerId; + } + + @Override + public void setMasterContainerId(ContainerId masterContainerId) { + maybeInitBuilder(); + if (masterContainerId == null) { + builder.clearMasterContainerId(); + } + this.masterContainerId = masterContainerId; + } + + public ApplicationAttemptStartDataProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.applicationAttemptId != null && !((ApplicationAttemptIdPBImpl) + this.applicationAttemptId).getProto().equals( + builder.getApplicationAttemptId())) { + builder.setApplicationAttemptId( + convertToProtoFormat(this.applicationAttemptId)); + } + if (this.masterContainerId != null && !((ContainerIdPBImpl) + this.masterContainerId).getProto().equals( + builder.getMasterContainerId())) { + builder.setMasterContainerId( + convertToProtoFormat(this.masterContainerId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationAttemptStartDataProto.newBuilder(proto); + } + viaProto = false; + } + + private ApplicationAttemptIdPBImpl convertFromProtoFormat( + ApplicationAttemptIdProto applicationAttemptId) { + return new ApplicationAttemptIdPBImpl(applicationAttemptId); + } + + private ApplicationAttemptIdProto convertToProtoFormat( + ApplicationAttemptId applicationAttemptId) { + return ((ApplicationAttemptIdPBImpl) applicationAttemptId).getProto(); + } + + private ContainerIdPBImpl convertFromProtoFormat( + ContainerIdProto containerId) { + return new ContainerIdPBImpl(containerId); + } + + private ContainerIdProto convertToProtoFormat( + ContainerId masterContainerId) { + return ((ContainerIdPBImpl) masterContainerId).getProto(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java new file mode 100644 index 0000000..a56f7e3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java @@ -0,0 +1,228 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationFinishDataProto; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationFinishDataProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; +import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData; + +import com.google.protobuf.TextFormat; + + +public class ApplicationFinishDataPBImpl + extends ApplicationFinishData { + + ApplicationFinishDataProto proto = + ApplicationFinishDataProto.getDefaultInstance(); + ApplicationFinishDataProto.Builder builder = null; + boolean viaProto = false; + + private ApplicationId applicationId; + + public ApplicationFinishDataPBImpl() { + builder = ApplicationFinishDataProto.newBuilder(); + } + + public ApplicationFinishDataPBImpl(ApplicationFinishDataProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public ApplicationId getApplicationId() { + if (this.applicationId != null) { + return this.applicationId; + } + ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationId()) { + return null; + } + this.applicationId = convertFromProtoFormat(p.getApplicationId()); + return this.applicationId; + } + + @Override + public void setApplicationId(ApplicationId applicationId) { + maybeInitBuilder(); + if (applicationId == null) { + builder.clearApplicationId(); + } + this.applicationId = applicationId; + } + + @Override + public long getFinishTime() { + ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getFinishTime(); + } + + @Override + public void setFinishTime(long finishTime) { + maybeInitBuilder(); + builder.setFinishTime(finishTime); + } + + @Override + public String getDiagnosticsInfo() { + ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasDiagnosticsInfo()) { + return null; + } + return p.getDiagnosticsInfo(); + } + + @Override + public void setDiagnosticsInfo(String diagnosticsInfo) { + maybeInitBuilder(); + if (diagnosticsInfo == null) { + builder.clearDiagnosticsInfo(); + return; + } + builder.setDiagnosticsInfo(diagnosticsInfo); + } + + @Override + public FinalApplicationStatus getFinalApplicationStatus() { + ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasFinalApplicationStatus()) { + return null; + } + return convertFromProtoFormat(p.getFinalApplicationStatus()); + } + + @Override + public void setFinalApplicationStatus( + FinalApplicationStatus finalApplicationStatus) { + maybeInitBuilder(); + if (finalApplicationStatus == null) { + builder.clearFinalApplicationStatus(); + return; + } + builder.setFinalApplicationStatus( + convertToProtoFormat(finalApplicationStatus)); + } + + @Override + public YarnApplicationState getYarnApplicationState() { + ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasYarnApplicationState()) { + return null; + } + return convertFromProtoFormat(p.getYarnApplicationState()); + } + + @Override + public void setYarnApplicationState(YarnApplicationState state) { + maybeInitBuilder(); + if (state == null) { + builder.clearYarnApplicationState(); + return; + } + builder.setYarnApplicationState(convertToProtoFormat(state)); + } + + public ApplicationFinishDataProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.applicationId != null && !((ApplicationIdPBImpl) + this.applicationId).getProto().equals(builder.getApplicationId())) { + builder.setApplicationId(convertToProtoFormat(this.applicationId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationFinishDataProto.newBuilder(proto); + } + viaProto = false; + } + + private ApplicationIdProto convertToProtoFormat( + ApplicationId applicationId) { + return ((ApplicationIdPBImpl) applicationId).getProto(); + } + + private ApplicationIdPBImpl convertFromProtoFormat( + ApplicationIdProto applicationId) { + return new ApplicationIdPBImpl(applicationId); + } + + private FinalApplicationStatus convertFromProtoFormat( + FinalApplicationStatusProto finalApplicationStatus) { + return ProtoUtils.convertFromProtoFormat(finalApplicationStatus); + } + + private FinalApplicationStatusProto convertToProtoFormat( + FinalApplicationStatus finalApplicationStatus) { + return ProtoUtils.convertToProtoFormat(finalApplicationStatus); + } + + private YarnApplicationStateProto convertToProtoFormat( + YarnApplicationState state) { + return ProtoUtils.convertToProtoFormat(state); + } + + private YarnApplicationState convertFromProtoFormat( + YarnApplicationStateProto yarnApplicationState) { + return ProtoUtils.convertFromProtoFormat(yarnApplicationState); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java deleted file mode 100644 index 7a65918..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb; - -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; -import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationHistoryDataProto; -import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationHistoryDataProtoOrBuilder; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; -import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; - - -public class ApplicationHistoryDataPBImpl - extends ProtoBase - implements ApplicationHistoryData { - - ApplicationHistoryDataProto proto = - ApplicationHistoryDataProto.getDefaultInstance(); - ApplicationHistoryDataProto.Builder builder = null; - boolean viaProto = false; - - private ApplicationId applicationId; - - public ApplicationHistoryDataPBImpl() { - builder = ApplicationHistoryDataProto.newBuilder(); - } - - public ApplicationHistoryDataPBImpl(ApplicationHistoryDataProto proto) { - this.proto = proto; - viaProto = true; - } - - @Override - public ApplicationId getApplicationId() { - if (this.applicationId != null) { - return this.applicationId; - } - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasApplicationId()) { - return null; - } - this.applicationId = convertFromProtoFormat(p.getApplicationId()); - return this.applicationId; - } - - @Override - public void setApplicationId(ApplicationId applicationId) { - maybeInitBuilder(); - if (applicationId == null) { - builder.clearApplicationId(); - } - this.applicationId = applicationId; - } - - @Override - public String getApplicationName() { - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasApplicationName()) { - return null; - } - return p.getApplicationName(); - } - - @Override - public void setApplicationName(String applicationName) { - maybeInitBuilder(); - if (applicationName == null) { - builder.clearApplicationName(); - return; - } - builder.setApplicationName(applicationName); - } - - @Override - public String getApplicationType() { - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasApplicationType()) { - return null; - } - return p.getApplicationType(); - } - - @Override - public void setApplicationType(String applicationType) { - maybeInitBuilder(); - if (applicationType == null) { - builder.clearApplicationType(); - return; - } - builder.setApplicationType(applicationType); - } - - @Override - public String getUser() { - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasUser()) { - return null; - } - return p.getUser(); - } - - @Override - public void setUser(String user) { - maybeInitBuilder(); - if (user == null) { - builder.clearUser(); - return; - } - builder.setUser(user); - } - - @Override - public String getQueue() { - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasQueue()) { - return null; - } - return p.getQueue(); - } - - @Override - public void setQueue(String queue) { - maybeInitBuilder(); - if (queue == null) { - builder.clearQueue(); - return; - } - builder.setQueue(queue); - } - - @Override - public long getSubmitTime() { - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getStartTime(); - } - - @Override - public void setSubmitTime(long submitTime) { - maybeInitBuilder(); - builder.setSubmitTime(submitTime); - } - - @Override - public long getStartTime() { - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getStartTime(); - } - - @Override - public void setStartTime(long startTime) { - maybeInitBuilder(); - builder.setStartTime(startTime); - } - - @Override - public long getFinishTime() { - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getFinishTime(); - } - - @Override - public void setFinishTime(long finishTime) { - maybeInitBuilder(); - builder.setFinishTime(finishTime); - } - - @Override - public String getDiagnosticsInfo() { - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasDiagnosticsInfo()) { - return null; - } - return p.getDiagnosticsInfo(); - } - - @Override - public void setDiagnosticsInfo(String diagnosticInfo) { - maybeInitBuilder(); - if (diagnosticInfo == null) { - builder.clearDiagnosticsInfo(); - return; - } - builder.setDiagnosticsInfo(diagnosticInfo); - } - - @Override - public FinalApplicationStatus getFinalApplicationStatus() { - ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasFinalApplicationStatus()) { - return null; - } - return convertFromProtoFormat(p.getFinalApplicationStatus()); - } - - @Override - public void setFinalApplicationStatus( - FinalApplicationStatus finalApplicationStatus) { - maybeInitBuilder(); - if (finalApplicationStatus == null) { - builder.clearFinalApplicationStatus(); - return; - } - builder.setFinalApplicationStatus( - convertToProtoFormat(finalApplicationStatus)); - } - - @Override - public ApplicationHistoryDataProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private void mergeLocalToBuilder() { - if (this.applicationId != null && !((ApplicationIdPBImpl) - this.applicationId).getProto().equals(builder.getApplicationId())) { - builder.setApplicationId(convertToProtoFormat(this.applicationId)); - } - } - - private void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = ApplicationHistoryDataProto.newBuilder(proto); - } - viaProto = false; - } - - private ApplicationIdProto convertToProtoFormat( - ApplicationId applicationId) { - return ((ApplicationIdPBImpl) applicationId).getProto(); - } - - private ApplicationIdPBImpl convertFromProtoFormat( - ApplicationIdProto applicationId) { - return new ApplicationIdPBImpl(applicationId); - } - - private FinalApplicationStatus convertFromProtoFormat( - FinalApplicationStatusProto finalApplicationStatus) { - return ProtoUtils.convertFromProtoFormat(finalApplicationStatus); - } - - private FinalApplicationStatusProto convertToProtoFormat( - FinalApplicationStatus finalApplicationStatus) { - return ProtoUtils.convertToProtoFormat(finalApplicationStatus); - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java new file mode 100644 index 0000000..b08023e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProto; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData; + +import com.google.protobuf.TextFormat; + + +public class ApplicationStartDataPBImpl + extends ApplicationStartData { + + ApplicationStartDataProto proto = + ApplicationStartDataProto.getDefaultInstance(); + ApplicationStartDataProto.Builder builder = null; + boolean viaProto = false; + + private ApplicationId applicationId; + + public ApplicationStartDataPBImpl() { + builder = ApplicationStartDataProto.newBuilder(); + } + + public ApplicationStartDataPBImpl(ApplicationStartDataProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public ApplicationId getApplicationId() { + if (this.applicationId != null) { + return this.applicationId; + } + ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationId()) { + return null; + } + this.applicationId = convertFromProtoFormat(p.getApplicationId()); + return this.applicationId; + } + + @Override + public void setApplicationId(ApplicationId applicationId) { + maybeInitBuilder(); + if (applicationId == null) { + builder.clearApplicationId(); + } + this.applicationId = applicationId; + } + + @Override + public String getApplicationName() { + ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationName()) { + return null; + } + return p.getApplicationName(); + } + + @Override + public void setApplicationName(String applicationName) { + maybeInitBuilder(); + if (applicationName == null) { + builder.clearApplicationName(); + return; + } + builder.setApplicationName(applicationName); + } + + @Override + public String getApplicationType() { + ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationType()) { + return null; + } + return p.getApplicationType(); + } + + @Override + public void setApplicationType(String applicationType) { + maybeInitBuilder(); + if (applicationType == null) { + builder.clearApplicationType(); + return; + } + builder.setApplicationType(applicationType); + } + + @Override + public String getUser() { + ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasUser()) { + return null; + } + return p.getUser(); + } + + @Override + public void setUser(String user) { + maybeInitBuilder(); + if (user == null) { + builder.clearUser(); + return; + } + builder.setUser(user); + } + + @Override + public String getQueue() { + ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasQueue()) { + return null; + } + return p.getQueue(); + } + + @Override + public void setQueue(String queue) { + maybeInitBuilder(); + if (queue == null) { + builder.clearQueue(); + return; + } + builder.setQueue(queue); + } + + @Override + public long getSubmitTime() { + ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getStartTime(); + } + + @Override + public void setSubmitTime(long submitTime) { + maybeInitBuilder(); + builder.setSubmitTime(submitTime); + } + + @Override + public long getStartTime() { + ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getStartTime(); + } + + @Override + public void setStartTime(long startTime) { + maybeInitBuilder(); + builder.setStartTime(startTime); + } + + public ApplicationStartDataProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.applicationId != null && !((ApplicationIdPBImpl) + this.applicationId).getProto().equals(builder.getApplicationId())) { + builder.setApplicationId(convertToProtoFormat(this.applicationId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationStartDataProto.newBuilder(proto); + } + viaProto = false; + } + + private ApplicationIdProto convertToProtoFormat( + ApplicationId applicationId) { + return ((ApplicationIdPBImpl) applicationId).getProto(); + } + + private ApplicationIdPBImpl convertFromProtoFormat( + ApplicationIdProto applicationId) { + return new ApplicationIdPBImpl(applicationId); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java new file mode 100644 index 0000000..12fef73 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerFinishDataProto; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerFinishDataProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData; + +import com.google.protobuf.TextFormat; + + +public class ContainerFinishDataPBImpl + extends ContainerFinishData { + + ContainerFinishDataProto proto = + ContainerFinishDataProto.getDefaultInstance(); + ContainerFinishDataProto.Builder builder = null; + boolean viaProto = false; + + private ContainerId containerId; + + public ContainerFinishDataPBImpl() { + builder = ContainerFinishDataProto.newBuilder(); + } + + public ContainerFinishDataPBImpl(ContainerFinishDataProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public ContainerId getContainerId() { + if (this.containerId != null) { + return this.containerId; + } + ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasContainerId()) { + return null; + } + this.containerId = convertFromProtoFormat(p.getContainerId()); + return this.containerId; + } + + @Override + public void setContainerId(ContainerId containerId) { + maybeInitBuilder(); + if (containerId == null) { + builder.clearContainerId(); + } + this.containerId = containerId; + } + + @Override + public long getFinishTime() { + ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getFinishTime(); + } + + @Override + public void setFinishTime(long finishTime) { + maybeInitBuilder(); + builder.setFinishTime(finishTime); + } + + @Override + public String getDiagnosticsInfo() { + ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasDiagnosticsInfo()) { + return null; + } + return p.getDiagnosticsInfo(); + } + + @Override + public void setDiagnosticsInfo(String diagnosticsInfo) { + maybeInitBuilder(); + if (diagnosticsInfo == null) { + builder.clearDiagnosticsInfo(); + return; + } + builder.setDiagnosticsInfo(diagnosticsInfo); + } + + @Override + public String getLogURL() { + ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasLogUrl()) { + return null; + } + return p.getLogUrl(); + } + + @Override + public void setLogURL(String logURL) { + maybeInitBuilder(); + if (logURL == null) { + builder.clearLogUrl(); + return; + } + builder.setLogUrl(logURL); + } + + @Override + public int getContainerExitStatus() { + ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getContainerExitStatus(); + } + + @Override + public ContainerState getContainerState() { + ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasContainerState()) { + return null; + } + return convertFromProtoFormat(p.getContainerState()); + } + + @Override + public void setContainerState(ContainerState state) { + maybeInitBuilder(); + if (state == null) { + builder.clearContainerState(); + return; + } + builder.setContainerState(convertToProtoFormat(state)); + } + + @Override + public void setContainerExitStatus(int containerExitStatus) { + maybeInitBuilder(); + builder.setContainerExitStatus(containerExitStatus); + } + + public ContainerFinishDataProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.containerId != null && !((ContainerIdPBImpl) + this.containerId).getProto().equals(builder.getContainerId())) { + builder.setContainerId(convertToProtoFormat(this.containerId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ContainerFinishDataProto.newBuilder(proto); + } + viaProto = false; + } + + private ContainerIdProto convertToProtoFormat( + ContainerId containerId) { + return ((ContainerIdPBImpl) containerId).getProto(); + } + + private ContainerIdPBImpl convertFromProtoFormat( + ContainerIdProto containerId) { + return new ContainerIdPBImpl(containerId); + } + + private ContainerStateProto convertToProtoFormat( + ContainerState state) { + return ProtoUtils.convertToProtoFormat(state); + } + + private ContainerState convertFromProtoFormat( + ContainerStateProto containerState) { + return ProtoUtils.convertFromProtoFormat(containerState); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java deleted file mode 100644 index 2cd95a8..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java +++ /dev/null @@ -1,343 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb; - -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; -import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; -import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerHistoryDataProto; -import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerHistoryDataProtoOrBuilder; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; -import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; - - -public class ContainerHistoryDataPBImpl - extends ProtoBase - implements ContainerHistoryData { - - ContainerHistoryDataProto proto = - ContainerHistoryDataProto.getDefaultInstance(); - ContainerHistoryDataProto.Builder builder = null; - boolean viaProto = false; - - private ContainerId containerId; - private Resource resource; - private NodeId nodeId; - private Priority priority; - - public ContainerHistoryDataPBImpl() { - builder = ContainerHistoryDataProto.newBuilder(); - } - - public ContainerHistoryDataPBImpl(ContainerHistoryDataProto proto) { - this.proto = proto; - viaProto = true; - } - - @Override - public ContainerId getContainerId() { - if (this != null) { - return this.containerId; - } - ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasContainerId()) { - return null; - } - this.containerId = convertFromProtoFormat(p.getContainerId()); - return this.containerId; - } - - @Override - public void setContainerId(ContainerId containerId) { - maybeInitBuilder(); - if (containerId == null) { - builder.clearContainerId(); - } - this.containerId = containerId; - } - - @Override - public Resource getAllocatedResource() { - if (this != null) { - return this.resource; - } - ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasAllocatedResource()) { - return null; - } - this.resource = convertFromProtoFormat(p.getAllocatedResource()); - return this.resource; - } - - @Override - public void setAllocatedResource(Resource resource) { - maybeInitBuilder(); - if (resource == null) { - builder.clearAllocatedResource(); - } - this.resource = resource; - } - - @Override - public NodeId getAssignedNode() { - if (this != null) { - return this.nodeId; - } - ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasAssignedNodeId()) { - return null; - } - this.nodeId = convertFromProtoFormat(p.getAssignedNodeId()); - return this.nodeId; - } - - @Override - public void setAssignedNode(NodeId nodeId) { - maybeInitBuilder(); - if (nodeId == null) { - builder.clearAssignedNodeId(); - } - this.nodeId = nodeId; - } - - @Override - public Priority getPriority() { - if (this != null) { - return this.priority; - } - ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasPriority()) { - return null; - } - this.priority = convertFromProtoFormat(p.getPriority()); - return this.priority; - } - - @Override - public void setPriority(Priority priority) { - maybeInitBuilder(); - if (priority == null) { - builder.clearPriority(); - } - this.priority = priority; - } - - @Override - public long getStartTime() { - ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getStartTime(); - } - - @Override - public void setStartTime(long startTime) { - maybeInitBuilder(); - builder.setStartTime(startTime); - } - - @Override - public long getFinishTime() { - ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getFinishTime(); - } - - @Override - public void setFinishTime(long finishTime) { - maybeInitBuilder(); - builder.setFinishTime(finishTime); - } - - @Override - public String getDiagnosticsInfo() { - ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasDiagnosticsInfo()) { - return null; - } - return p.getDiagnosticsInfo(); - } - - @Override - public void setDiagnosticsInfo(String diagnosticInfo) { - maybeInitBuilder(); - if (diagnosticInfo == null) { - builder.clearDiagnosticsInfo(); - return; - } - builder.setDiagnosticsInfo(diagnosticInfo); - } - - @Override - public String getLogURL() { - ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasLogUrl()) { - return null; - } - return p.getLogUrl(); - } - - @Override - public void setLogURL(String logURL) { - maybeInitBuilder(); - if (logURL == null) { - builder.clearLogUrl(); - return; - } - builder.setLogUrl(logURL); - } - - @Override - public ContainerState getFinalContainerStatus() { - ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasFinalContainerStatus()) { - return null; - } - return convertFromProtoFormat(p.getFinalContainerStatus()); - } - - @Override - public void setFinalContainerStatus( - ContainerState finalContainerState) { - maybeInitBuilder(); - if (finalContainerState == null) { - builder.clearFinalContainerStatus(); - return; - } - builder.setFinalContainerStatus( - convertToProtoFormat(finalContainerState)); - } - - @Override - public ContainerHistoryDataProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private void mergeLocalToBuilder() { - if (this.containerId != null && !((ContainerIdPBImpl) - this.containerId).getProto().equals(builder.getContainerId())) { - builder.setContainerId(convertToProtoFormat(this.containerId)); - } - if (this.resource != null && !((ResourcePBImpl) - this.resource).getProto().equals(builder.getAllocatedResource())) { - builder.setAllocatedResource(convertToProtoFormat(this.resource)); - } - if (this.nodeId != null && !((NodeIdPBImpl) - this.nodeId).getProto().equals(builder.getAssignedNodeId())) { - builder.setAssignedNodeId(convertToProtoFormat(this.nodeId)); - } - if (this.priority != null && !((PriorityPBImpl) - this.priority).getProto().equals(builder.getPriority())) { - builder.setPriority(convertToProtoFormat(this.priority)); - } - } - - private void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = ContainerHistoryDataProto.newBuilder(proto); - } - viaProto = false; - } - - private ContainerIdProto convertToProtoFormat( - ContainerId ContainerId) { - return ((ContainerIdPBImpl) containerId).getProto(); - } - - private ContainerIdPBImpl convertFromProtoFormat( - ContainerIdProto containerId) { - return new ContainerIdPBImpl(containerId); - } - - private ResourceProto convertToProtoFormat(Resource resource) { - return ((ResourcePBImpl) resource).getProto(); - } - - private ResourcePBImpl convertFromProtoFormat(ResourceProto resource) { - return new ResourcePBImpl(resource); - } - - private NodeIdProto convertToProtoFormat(NodeId nodeId) { - return ((NodeIdPBImpl) nodeId).getProto(); - } - - private NodeIdPBImpl convertFromProtoFormat(NodeIdProto nodeId) { - return new NodeIdPBImpl(nodeId); - } - - private PriorityProto convertToProtoFormat(Priority priority) { - return ((PriorityPBImpl) priority).getProto(); - } - - private PriorityPBImpl convertFromProtoFormat(PriorityProto priority) { - return new PriorityPBImpl(priority); - } - - private ContainerState convertFromProtoFormat( - ContainerStateProto finalContainerStatus) { - return ProtoUtils.convertFromProtoFormat(finalContainerStatus); - } - - private ContainerStateProto convertToProtoFormat( - ContainerState finalContainerStatus) { - return ProtoUtils.convertToProtoFormat(finalContainerStatus); - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java new file mode 100644 index 0000000..b0b0cb4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java @@ -0,0 +1,258 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProto; +import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData; + +import com.google.protobuf.TextFormat; + + +public class ContainerStartDataPBImpl + extends ContainerStartData { + + ContainerStartDataProto proto = + ContainerStartDataProto.getDefaultInstance(); + ContainerStartDataProto.Builder builder = null; + boolean viaProto = false; + + private ContainerId containerId; + private Resource resource; + private NodeId nodeId; + private Priority priority; + + public ContainerStartDataPBImpl() { + builder = ContainerStartDataProto.newBuilder(); + } + + public ContainerStartDataPBImpl(ContainerStartDataProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public ContainerId getContainerId() { + if (this.containerId != null) { + return this.containerId; + } + ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasContainerId()) { + return null; + } + this.containerId = convertFromProtoFormat(p.getContainerId()); + return this.containerId; + } + + @Override + public void setContainerId(ContainerId containerId) { + maybeInitBuilder(); + if (containerId == null) { + builder.clearContainerId(); + } + this.containerId = containerId; + } + + @Override + public Resource getAllocatedResource() { + if (this.resource != null) { + return this.resource; + } + ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasAllocatedResource()) { + return null; + } + this.resource = convertFromProtoFormat(p.getAllocatedResource()); + return this.resource; + } + + @Override + public void setAllocatedResource(Resource resource) { + maybeInitBuilder(); + if (resource == null) { + builder.clearAllocatedResource(); + } + this.resource = resource; + } + + @Override + public NodeId getAssignedNode() { + if (this.nodeId != null) { + return this.nodeId; + } + ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasAssignedNodeId()) { + return null; + } + this.nodeId = convertFromProtoFormat(p.getAssignedNodeId()); + return this.nodeId; + } + + @Override + public void setAssignedNode(NodeId nodeId) { + maybeInitBuilder(); + if (nodeId == null) { + builder.clearAssignedNodeId(); + } + this.nodeId = nodeId; + } + + @Override + public Priority getPriority() { + if (this.priority != null) { + return this.priority; + } + ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasPriority()) { + return null; + } + this.priority = convertFromProtoFormat(p.getPriority()); + return this.priority; + } + + @Override + public void setPriority(Priority priority) { + maybeInitBuilder(); + if (priority == null) { + builder.clearPriority(); + } + this.priority = priority; + } + + @Override + public long getStartTime() { + ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getStartTime(); + } + + @Override + public void setStartTime(long startTime) { + maybeInitBuilder(); + builder.setStartTime(startTime); + } + + public ContainerStartDataProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.containerId != null && !((ContainerIdPBImpl) + this.containerId).getProto().equals(builder.getContainerId())) { + builder.setContainerId(convertToProtoFormat(this.containerId)); + } + if (this.resource != null && !((ResourcePBImpl) + this.resource).getProto().equals(builder.getAllocatedResource())) { + builder.setAllocatedResource(convertToProtoFormat(this.resource)); + } + if (this.nodeId != null && !((NodeIdPBImpl) + this.nodeId).getProto().equals(builder.getAssignedNodeId())) { + builder.setAssignedNodeId(convertToProtoFormat(this.nodeId)); + } + if (this.priority != null && !((PriorityPBImpl) + this.priority).getProto().equals(builder.getPriority())) { + builder.setPriority(convertToProtoFormat(this.priority)); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ContainerStartDataProto.newBuilder(proto); + } + viaProto = false; + } + + private ContainerIdProto convertToProtoFormat( + ContainerId containerId) { + return ((ContainerIdPBImpl) containerId).getProto(); + } + + private ContainerIdPBImpl convertFromProtoFormat( + ContainerIdProto containerId) { + return new ContainerIdPBImpl(containerId); + } + + private ResourceProto convertToProtoFormat(Resource resource) { + return ((ResourcePBImpl) resource).getProto(); + } + + private ResourcePBImpl convertFromProtoFormat(ResourceProto resource) { + return new ResourcePBImpl(resource); + } + + private NodeIdProto convertToProtoFormat(NodeId nodeId) { + return ((NodeIdPBImpl) nodeId).getProto(); + } + + private NodeIdPBImpl convertFromProtoFormat(NodeIdProto nodeId) { + return new NodeIdPBImpl(nodeId); + } + + private PriorityProto convertToProtoFormat(Priority priority) { + return ((PriorityPBImpl) priority).getProto(); + } + + private PriorityPBImpl convertFromProtoFormat(PriorityProto priority) { + return new PriorityPBImpl(priority); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java new file mode 100644 index 0000000..4eafdb2 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import java.io.IOException; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData; + +public class ApplicationHistoryStoreTestUtils { + + protected ApplicationHistoryStore store; + + protected void writeApplicationStartData(ApplicationId appId) + throws IOException { + store.applicationStarted( + ApplicationStartData.newInstance(appId, appId.toString(), "test type", + "test queue", + "test user", 0, 0)); + } + + protected void writeApplicationFinishData(ApplicationId appId) + throws IOException { + store.applicationFinished( + ApplicationFinishData.newInstance( + appId, 0, appId.toString(), FinalApplicationStatus.UNDEFINED, + YarnApplicationState.FINISHED)); + } + + protected void writeApplicationAttemptStartData( + ApplicationAttemptId appAttemptId) throws IOException { + store.applicationAttemptStarted( + ApplicationAttemptStartData.newInstance( + appAttemptId, appAttemptId.toString(), 0, + ContainerId.newInstance(appAttemptId, 1))); + } + + protected void writeApplicationAttemptFinishData( + ApplicationAttemptId appAttemptId) throws IOException { + store.applicationAttemptFinished( + ApplicationAttemptFinishData.newInstance(appAttemptId, + appAttemptId.toString(), "test diagnostics info", + FinalApplicationStatus.UNDEFINED, + YarnApplicationAttemptState.FINISHED)); + } + + protected void writeContainerStartData(ContainerId containerId) + throws IOException { + store.containerStarted( + ContainerStartData.newInstance(containerId, Resource.newInstance(0, 0), + NodeId.newInstance("localhost", 0), + Priority.newInstance(containerId.getId()), 0)); + } + + protected void writeContainerFinishData(ContainerId containerId) + throws IOException { + store.containerFinished( + ContainerFinishData.newInstance(containerId, 0, containerId.toString(), + "http://localhost:0/", 0, ContainerState.COMPLETE)); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java index 82a764e..51c01ff 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java @@ -19,102 +19,189 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; import java.io.IOException; -import java.util.HashMap; import junit.framework.Assert; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; -import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationAttemptHistoryDataPBImpl; -import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationHistoryDataPBImpl; -import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ContainerHistoryDataPBImpl; -import org.junit.After; import org.junit.Before; import org.junit.Test; -public class TestMemoryApplicationHistoryStore { - MemoryApplicationHistoryStore memstore = null; +public class TestMemoryApplicationHistoryStore extends + ApplicationHistoryStoreTestUtils { @Before - public void setup() throws Throwable { - memstore = MemoryApplicationHistoryStore.getMemoryStore(); - writeHistoryApplication(); - writeHistoryApplicationAttempt(); - writeContainer(); + public void setup() { + store = new MemoryApplicationHistoryStore(); } - public void writeHistoryApplication() throws Throwable { - ApplicationHistoryData appData = new ApplicationHistoryDataPBImpl(); - appData.setApplicationId(ApplicationId.newInstance(1234, 1)); - memstore.writeApplication(appData); - } - - public void writeHistoryApplicationAttempt() throws Throwable { - ApplicationAttemptHistoryData appAttemptHistoryData = - new ApplicationAttemptHistoryDataPBImpl(); - appAttemptHistoryData.setApplicationAttemptId(ApplicationAttemptId - .newInstance(ApplicationId.newInstance(1234, 1), 1)); - memstore.writeApplicationAttempt(appAttemptHistoryData); + @Test + public void testReadWriteApplicationHistory() throws Exception { + // Out of order + ApplicationId appId = ApplicationId.newInstance(0, 1); + try { + writeApplicationFinishData(appId); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains( + "is stored before the start information")); + } + // Normal + int numApps = 5; + for (int i = 1; i <= numApps; ++i) { + appId = ApplicationId.newInstance(0, i); + writeApplicationStartData(appId); + writeApplicationFinishData(appId); + } + Assert.assertEquals(numApps, store.getAllApplications().size()); + for (int i = 1; i <= numApps; ++i) { + appId = ApplicationId.newInstance(0, i); + ApplicationHistoryData data = store.getApplication(appId); + Assert.assertNotNull(data); + Assert.assertEquals(appId.toString(), data.getApplicationName()); + Assert.assertEquals(appId.toString(), data.getDiagnosticsInfo()); + } + // Write again + appId = ApplicationId.newInstance(0, 1); + try { + writeApplicationStartData(appId); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("is already stored")); + } + try { + writeApplicationFinishData(appId); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("is already stored")); + } } - public void writeContainer() throws Throwable { - ContainerHistoryData container = new ContainerHistoryDataPBImpl(); - container.setContainerId(ContainerId.newInstance(ApplicationAttemptId - .newInstance(ApplicationId.newInstance(1234, 1), 1), 1)); - memstore.writeContainer(container); + @Test + public void testReadWriteApplicationAttemptHistory() throws Exception { + // Out of order + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + try { + writeApplicationAttemptFinishData(appAttemptId); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains( + "is stored before the start information")); + } + // Normal + int numAppAttempts = 5; + writeApplicationStartData(appId); + for (int i = 1; i <= numAppAttempts; ++i) { + appAttemptId = + ApplicationAttemptId.newInstance(appId, i); + writeApplicationAttemptStartData(appAttemptId); + writeApplicationAttemptFinishData(appAttemptId); + } + Assert.assertEquals( + numAppAttempts, store.getApplicationAttempts(appId).size()); + for (int i = 1; i <= numAppAttempts; ++i) { + appAttemptId = + ApplicationAttemptId.newInstance(appId, i); + ApplicationAttemptHistoryData data = + store.getApplicationAttempt(appAttemptId); + Assert.assertNotNull(data); + Assert.assertEquals(appAttemptId.toString(), data.getHost()); + Assert.assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo()); + } + writeApplicationFinishData(appId); + // Write again + appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + try { + writeApplicationAttemptStartData(appAttemptId); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("is already stored")); + } + try { + writeApplicationAttemptFinishData(appAttemptId); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("is already stored")); + } } - public ContainerHistoryData writeContainer(ApplicationAttemptId appAttemptId, - int containerId) throws Throwable { - ContainerHistoryData container = new ContainerHistoryDataPBImpl(); - container - .setContainerId(ContainerId.newInstance(appAttemptId, containerId)); - memstore.writeContainer(container); - return container; - } - - @After - public void tearDown() { + @Test + public void testReadWriteContainerHistory() throws Exception { + // Out of order + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId.newInstance(appAttemptId, 1); + try { + writeContainerFinishData(containerId); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains( + "is stored before the start information")); + } + // Normal + writeApplicationAttemptStartData(appAttemptId); + int numContainers = 5; + for (int i = 1; i <= numContainers; ++i) { + containerId = ContainerId.newInstance(appAttemptId, i); + writeContainerStartData(containerId); + writeContainerFinishData(containerId); + } + Assert.assertEquals( + numContainers, store.getContainers(appAttemptId).size()); + for (int i = 1; i <= numContainers; ++i) { + containerId = ContainerId.newInstance(appAttemptId, i); + ContainerHistoryData data = store.getContainer(containerId); + Assert.assertNotNull(data); + Assert.assertEquals(Priority.newInstance(containerId.getId()), + data.getPriority()); + Assert.assertEquals(containerId.toString(), data.getDiagnosticsInfo()); + } + ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId); + Assert.assertNotNull(masterContainer); + Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1), + masterContainer.getContainerId()); + writeApplicationAttemptFinishData(appAttemptId); + // Write again + containerId = ContainerId.newInstance(appAttemptId, 1); + try { + writeContainerStartData(containerId); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("is already stored")); + } + try { + writeContainerFinishData(containerId); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("is already stored")); + } } @Test - public void testReadApplication() throws Throwable { - HashMap map = - (HashMap) memstore - .getAllApplications(); - Assert.assertEquals(1, map.size()); - ApplicationHistoryData appData = null; - for (ApplicationId appId : map.keySet()) { - appData = map.get(appId); - Assert.assertEquals("application_1234_0001", appData.getApplicationId() - .toString()); - } - HashMap appAttempts = - (HashMap) memstore - .getApplicationAttempts(appData.getApplicationId()); - Assert.assertEquals(1, appAttempts.size()); - ApplicationAttemptHistoryData appAttempt = null; - for (ApplicationAttemptId appAttemptId : appAttempts.keySet()) { - appAttempt = appAttempts.get(appAttemptId); - Assert.assertEquals("appattempt_1234_0001_000001", appAttempt - .getApplicationAttemptId().toString()); - } - ContainerHistoryData amContainer = - memstore.getContainer(ContainerId.newInstance(ApplicationAttemptId - .newInstance(ApplicationId.newInstance(1234, 1), 1), 1)); - Assert.assertEquals("container_1234_0001_01_000001", amContainer - .getContainerId().toString()); - ContainerHistoryData container2 = - writeContainer(appAttempt.getApplicationAttemptId(), 2); - HashMap containers = - (HashMap) memstore - .getContainers(appAttempt.getApplicationAttemptId()); - Assert.assertEquals(2, containers.size()); - Assert.assertEquals("container_1234_0001_01_000002", containers.get( - container2.getContainerId()).getContainerId().toString()); + public void testMassiveWriteContainerHistory() throws IOException { + long mb = 1024 * 1024; + Runtime runtime = Runtime.getRuntime(); + long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb; + int numContainers = 100000; + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + for (int i = 1; i <= numContainers; ++i) { + ContainerId containerId = ContainerId.newInstance(appAttemptId, i); + writeContainerStartData(containerId); + writeContainerFinishData(containerId); + } + long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb; + Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 100); } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 8c73443..5556255 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -33,9 +33,14 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException; import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; @@ -43,6 +48,7 @@ * Utility methods to aid serving RM data through the REST and RPC APIs */ public class RMServerUtils { + public static List queryRMNodes(RMContext context, EnumSet acceptedStates) { // nodes contains nodes that are NEW, RUNNING OR UNHEALTHY @@ -56,7 +62,7 @@ } } } - + // inactiveNodes contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED if (acceptedStates.contains(NodeState.DECOMMISSIONED) || acceptedStates.contains(NodeState.LOST) || @@ -69,7 +75,7 @@ } return results; } - + /** * Utility method to validate a list resource requests, by insuring that the * requested memory/vcore is non-negative and not greater than max @@ -85,8 +91,9 @@ public static void validateResourceRequests(List ask, * @throw InvalidResourceBlacklistRequestException if the * resource is not able to be added to the blacklist. */ - public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest) - throws InvalidResourceBlacklistRequestException { + public static void validateBlacklistRequest( + ResourceBlacklistRequest blacklistRequest) + throws InvalidResourceBlacklistRequestException { if (blacklistRequest != null) { List plus = blacklistRequest.getBlacklistAdditions(); if (plus != null && plus.contains(ResourceRequest.ANY)) { @@ -100,10 +107,12 @@ public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRe * It will validate to make sure all the containers belong to correct * application attempt id. If not then it will throw * {@link InvalidContainerReleaseException} - * @param containerReleaseList containers to be released as requested by - * application master. - * @param appAttemptId Application attempt Id - * @throws InvalidContainerReleaseException + * + * @param containerReleaseList + * containers to be released as requested by application master. + * @param appAttemptId + * Application attempt Id + * @throws InvalidContainerReleaseException */ public static void validateContainerReleaseRequest(List containerReleaseList, @@ -111,9 +120,11 @@ public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRe throws InvalidContainerReleaseException { for (ContainerId cId : containerReleaseList) { if (!appAttemptId.equals(cId.getApplicationAttemptId())) { - throw new InvalidContainerReleaseException("Cannot release container : " - + cId.toString() + " not belonging to this application attempt : " - + appAttemptId); + throw new InvalidContainerReleaseException( + "Cannot release container : " + + cId.toString() + + " not belonging to this application attempt : " + + appAttemptId); } } } @@ -157,4 +168,63 @@ public static UserGroupInformation verifyAccess( } return user; } + + public static YarnApplicationState createApplicationState( + RMAppState rmAppState) { + switch (rmAppState) { + case NEW: + return YarnApplicationState.NEW; + case NEW_SAVING: + return YarnApplicationState.NEW_SAVING; + case SUBMITTED: + return YarnApplicationState.SUBMITTED; + case ACCEPTED: + return YarnApplicationState.ACCEPTED; + case RUNNING: + return YarnApplicationState.RUNNING; + case FINISHING: + case FINISHED: + return YarnApplicationState.FINISHED; + case KILLED: + return YarnApplicationState.KILLED; + case FAILED: + return YarnApplicationState.FAILED; + default: + throw new YarnRuntimeException("Unknown state passed!"); + } + } + + public static YarnApplicationAttemptState createApplicationAttemptState( + RMAppAttemptState rmAppAttemptState) { + switch (rmAppAttemptState) { + case NEW: + return YarnApplicationAttemptState.NEW; + case SUBMITTED: + return YarnApplicationAttemptState.SUBMITTED; + case SCHEDULED: + return YarnApplicationAttemptState.SCHEDULED; + case ALLOCATED: + return YarnApplicationAttemptState.ALLOCATED; + case LAUNCHED: + return YarnApplicationAttemptState.LAUNCHED; + case ALLOCATED_SAVING: + case LAUNCHED_UNMANAGED_SAVING: + return YarnApplicationAttemptState.ALLOCATED_SAVING; + case RUNNING: + return YarnApplicationAttemptState.RUNNING; + case FINISHING: + return YarnApplicationAttemptState.FINISHING; + case FINAL_SAVING: + return YarnApplicationAttemptState.FINAL_SAVING; + case FINISHED: + return YarnApplicationAttemptState.FINISHED; + case KILLED: + return YarnApplicationAttemptState.KILLED; + case FAILED: + return YarnApplicationAttemptState.FAILED; + default: + throw new YarnRuntimeException("Unknown state passed!"); + } + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 1d451fb..4457a88 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; @@ -1056,27 +1057,7 @@ public YarnApplicationState createApplicationState() { if (rmAppState.equals(RMAppState.KILLING)) { rmAppState = stateBeforeKilling; } - switch (rmAppState) { - case NEW: - return YarnApplicationState.NEW; - case NEW_SAVING: - return YarnApplicationState.NEW_SAVING; - case SUBMITTED: - return YarnApplicationState.SUBMITTED; - case ACCEPTED: - return YarnApplicationState.ACCEPTED; - case RUNNING: - return YarnApplicationState.RUNNING; - case FINISHING: - case FINISHED: - return YarnApplicationState.FINISHED; - case KILLED: - return YarnApplicationState.KILLED; - case FAILED: - return YarnApplicationState.FAILED; - default: - throw new YarnRuntimeException("Unknown state passed!"); - } + return RMServerUtils.createApplicationState(rmAppState); } public static boolean isAppInFinalState(RMApp rmApp) {