diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 44c35c3..7ee53af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -261,6 +261,16 @@ public static final String FS_RM_STATE_STORE_URI = RM_PREFIX + "fs.rm-state-store.uri"; + /** The setting that controls whether RM writes history data.*/ + public static final String RM_AHS_WRITER_ENABLED = + RM_PREFIX + "ahs.writer.enabled"; + + public static final boolean DEFAULT_RM_AHS_WRITER_ENABLED = false; + + /** The class that persists the history data.*/ + public static final String RM_AHS_WRITER_CLASS = + RM_PREFIX + "ahs.writer.class"; + /** The maximum number of completed applications RM keeps. */ public static final String RM_MAX_COMPLETED_APPLICATIONS = RM_PREFIX + "max-completed-applications"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index b6753bc..d140de9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -255,6 +255,19 @@ + Enable RM to write history data. If true, then + yarn.resourcemanager.ahs.writer.class must be specified + yarn.resourcemanager.ahs.writer.enabled + false + + + + The class that persists the history data. + yarn.resourcemanager.ahs.writer.class + org.apache.hadoop.yarn.server.applicationhistoryservice.DummyApplicationHistoryWriter + + + The maximum number of completed applications RM keeps. yarn.resourcemanager.max-completed-applications 10000 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index ed04555..8b310d9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -26,9 +26,9 @@ 4.0.0 org.apache.hadoop - hadoop-yarn-server-applicationhistoryserver + hadoop-yarn-server-applicationhistoryservice 2.2.0-SNAPSHOT - hadoop-yarn-server-applicationhistoryserver + hadoop-yarn-server-applicationhistoryservice diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/DummyApplicationHistoryWriter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/DummyApplicationHistoryWriter.java new file mode 100644 index 0000000..2a7fa8d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/DummyApplicationHistoryWriter.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; + +/** + * The dummy implementation of {@link ApplicationHistoryWriter}. If this + * implementation is used, no history data will be persisted. + * + */ +@Unstable +@Private +public class DummyApplicationHistoryWriter implements ApplicationHistoryWriter { + + @Override + public void writeApplication(ApplicationHistoryData app) throws Throwable { + // Do nothing + } + + @Override + public void writeApplicationAttempt(ApplicationAttemptHistoryData appAttempt) + throws Throwable { + // Do nothing + } + + @Override + public void writeContainer(ContainerHistoryData container) throws Throwable { + // Do nothing + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java index 0c3afa3..9fe7b06 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.util.Records; /** * The class contains all the fields that need to be stored persistently for @@ -30,62 +31,82 @@ */ @Public @Unstable -public interface ApplicationAttemptHistoryData { +public abstract class ApplicationAttemptHistoryData { @Public @Unstable - ApplicationAttemptId getApplicationAttemptId(); + public static ApplicationAttemptHistoryData newInstance( + ApplicationAttemptId appAttemptId, String host, int rpcPort, + ContainerId masterContainerId, String diagnosticsInfo, String trackingURL, + FinalApplicationStatus finalApplicationStatus) { + ApplicationAttemptHistoryData appAttemptHD = + Records.newRecord(ApplicationAttemptHistoryData.class); + appAttemptHD.setApplicationAttemptId(appAttemptId); + appAttemptHD.setHost(host); + appAttemptHD.setRPCPort(rpcPort); + appAttemptHD.setMasterContainerId(masterContainerId); + appAttemptHD.setDiagnosticsInfo(diagnosticsInfo); + appAttemptHD.setTrackingURL(trackingURL); + appAttemptHD.setFinalApplicationStatus(finalApplicationStatus); + return appAttemptHD; + } @Public @Unstable - void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId); + public abstract ApplicationAttemptId getApplicationAttemptId(); @Public @Unstable - String getHost(); + public abstract void setApplicationAttemptId( + ApplicationAttemptId applicationAttemptId); @Public @Unstable - void setHost(String host); + public abstract String getHost(); @Public @Unstable - int getRPCPort(); + public abstract void setHost(String host); @Public @Unstable - void setRPCPort(int rpcPort); + public abstract int getRPCPort(); @Public @Unstable - String getTrackingURL(); + public abstract void setRPCPort(int rpcPort); @Public @Unstable - void setTrackingURL(String trackingURL); + public abstract String getTrackingURL(); @Public @Unstable - String getDiagnosticsInfo(); + public abstract void setTrackingURL(String trackingURL); @Public @Unstable - void setDiagnosticsInfo(String diagnosticsInfo); + public abstract String getDiagnosticsInfo(); @Public @Unstable - FinalApplicationStatus getFinalApplicationStatus(); + public abstract void setDiagnosticsInfo(String diagnosticsInfo); @Public @Unstable - void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus); + public abstract FinalApplicationStatus getFinalApplicationStatus(); @Public @Unstable - ContainerId getMasterContainerId(); + public abstract void setFinalApplicationStatus( + FinalApplicationStatus finalApplicationStatus); @Public @Unstable - void setMasterContainerId(ContainerId masterContainerId); + public abstract ContainerId getMasterContainerId(); + + @Public + @Unstable + public abstract void setMasterContainerId(ContainerId masterContainerId); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java index 1512864..257b55a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.util.Records; /** * The class contains all the fields that need to be stored persistently for @@ -29,86 +30,108 @@ */ @Public @Unstable -public interface ApplicationHistoryData { +public abstract class ApplicationHistoryData { @Public @Unstable - ApplicationId getApplicationId(); + public static ApplicationHistoryData newInstance(ApplicationId applicationId, + String applicationName, String applicationType, String queue, String user, + long submitTime, long startTime, long finishTime, String diagnosticsInfo, + FinalApplicationStatus finalApplicationStatus) { + ApplicationHistoryData appHD = + Records.newRecord(ApplicationHistoryData.class); + appHD.setApplicationId(applicationId); + appHD.setApplicationName(applicationName); + appHD.setApplicationType(applicationType); + appHD.setQueue(queue); + appHD.setUser(user); + appHD.setSubmitTime(submitTime); + appHD.setStartTime(startTime); + appHD.setFinishTime(finishTime); + appHD.setDiagnosticsInfo(diagnosticsInfo); + appHD.setFinalApplicationStatus(finalApplicationStatus); + return appHD; + } @Public @Unstable - void setApplicationId(ApplicationId applicationId); + public abstract ApplicationId getApplicationId(); @Public @Unstable - String getApplicationName(); + public abstract void setApplicationId(ApplicationId applicationId); @Public @Unstable - void setApplicationName(String applicationName); + public abstract String getApplicationName(); @Public @Unstable - String getApplicationType(); + public abstract void setApplicationName(String applicationName); @Public @Unstable - void setApplicationType(String applicationType); + public abstract String getApplicationType(); @Public @Unstable - String getUser(); + public abstract void setApplicationType(String applicationType); @Public @Unstable - void setUser(String user); + public abstract String getUser(); @Public @Unstable - String getQueue(); + public abstract void setUser(String user); @Public @Unstable - void setQueue(String queue); + public abstract String getQueue(); @Public @Unstable - long getSubmitTime(); + public abstract void setQueue(String queue); @Public @Unstable - void setSubmitTime(long submitTime); + public abstract long getSubmitTime(); @Public @Unstable - long getStartTime(); + public abstract void setSubmitTime(long submitTime); @Public @Unstable - void setStartTime(long startTime); + public abstract long getStartTime(); @Public @Unstable - long getFinishTime(); + public abstract void setStartTime(long startTime); @Public @Unstable - void setFinishTime(long finishTime); + public abstract long getFinishTime(); @Public @Unstable - String getDiagnosticsInfo(); + public abstract void setFinishTime(long finishTime); @Public @Unstable - void setDiagnosticsInfo(String diagnosticInfo); + public abstract String getDiagnosticsInfo(); @Public @Unstable - FinalApplicationStatus getFinalApplicationStatus(); + public abstract void setDiagnosticsInfo(String diagnosticInfo); @Public @Unstable - void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus); + public abstract FinalApplicationStatus getFinalApplicationStatus(); + + @Public + @Unstable + public abstract void setFinalApplicationStatus( + FinalApplicationStatus finalApplicationStatus); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java index f069b5c..83fc71d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.Records; /** * The class contains all the fields that need to be stored persistently for @@ -32,78 +33,99 @@ */ @Public @Unstable -public interface ContainerHistoryData { +public abstract class ContainerHistoryData { @Public @Unstable - ContainerId getContainerId(); + public static ContainerHistoryData newInstance(ContainerId containerId, + Resource allocatedResource, NodeId assignedNode, Priority priority, + long startTime, long finishTime, String diagnosticsInfo, String logURL, + ContainerState finalContainerStatus) { + ContainerHistoryData containerHD = + Records.newRecord(ContainerHistoryData.class); + containerHD.setContainerId(containerId); + containerHD.setAllocatedResource(allocatedResource); + containerHD.setAssignedNode(assignedNode); + containerHD.setPriority(priority); + containerHD.setStartTime(startTime); + containerHD.setFinishTime(finishTime); + containerHD.setDiagnosticsInfo(diagnosticsInfo); + containerHD.setLogURL(logURL); + containerHD.setFinalContainerStatus(finalContainerStatus); + return containerHD; + } @Public @Unstable - void setContainerId(ContainerId containerId); + public abstract ContainerId getContainerId(); @Public @Unstable - Resource getAllocatedResource(); + public abstract void setContainerId(ContainerId containerId); @Public @Unstable - void setAllocatedResource(Resource resource); + public abstract Resource getAllocatedResource(); @Public @Unstable - NodeId getAssignedNode(); + public abstract void setAllocatedResource(Resource resource); @Public @Unstable - void setAssignedNode(NodeId nodeId); + public abstract NodeId getAssignedNode(); @Public @Unstable - Priority getPriority(); + public abstract void setAssignedNode(NodeId nodeId); @Public @Unstable - void setPriority(Priority priority); + public abstract Priority getPriority(); @Public @Unstable - long getStartTime(); + public abstract void setPriority(Priority priority); @Public @Unstable - void setStartTime(long startTime); + public abstract long getStartTime(); @Public @Unstable - long getFinishTime(); + public abstract void setStartTime(long startTime); @Public @Unstable - void setFinishTime(long finishTime); + public abstract long getFinishTime(); @Public @Unstable - String getDiagnosticsInfo(); + public abstract void setFinishTime(long finishTime); @Public @Unstable - void setDiagnosticsInfo(String diagnosticInfo); + public abstract String getDiagnosticsInfo(); @Public @Unstable - String getLogURL(); + public abstract void setDiagnosticsInfo(String diagnosticInfo); @Public @Unstable - void setLogURL(String logURL); + public abstract String getLogURL(); @Public @Unstable - ContainerState getFinalContainerStatus(); + public abstract void setLogURL(String logURL); @Public @Unstable - void setFinalContainerStatus(ContainerState finalContainerState); + public abstract ContainerState getFinalContainerStatus(); + + @Public + @Unstable + public abstract void setFinalContainerStatus( + ContainerState finalContainerState); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java index 3362875..a25d7a1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java @@ -23,7 +23,6 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptHistoryDataProto; import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptHistoryDataProtoOrBuilder; @@ -34,8 +33,7 @@ public class ApplicationAttemptHistoryDataPBImpl - extends ProtoBase - implements ApplicationAttemptHistoryData { + extends ApplicationAttemptHistoryData { ApplicationAttemptHistoryDataProto proto = ApplicationAttemptHistoryDataProto.getDefaultInstance(); @@ -192,7 +190,6 @@ public void setMasterContainerId(ContainerId masterContainerId) { this.masterContainerId = masterContainerId; } - @Override public ApplicationAttemptHistoryDataProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java index 7a65918..a140ef8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java @@ -21,7 +21,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationHistoryDataProto; import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationHistoryDataProtoOrBuilder; @@ -30,9 +29,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; -public class ApplicationHistoryDataPBImpl - extends ProtoBase - implements ApplicationHistoryData { +public class ApplicationHistoryDataPBImpl extends ApplicationHistoryData { ApplicationHistoryDataProto proto = ApplicationHistoryDataProto.getDefaultInstance(); @@ -224,7 +221,6 @@ public void setFinalApplicationStatus( convertToProtoFormat(finalApplicationStatus)); } - @Override public ApplicationHistoryDataProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java index 2cd95a8..33dd852 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java @@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerHistoryDataProto; @@ -39,9 +38,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; -public class ContainerHistoryDataPBImpl - extends ProtoBase - implements ContainerHistoryData { +public class ContainerHistoryDataPBImpl extends ContainerHistoryData { ContainerHistoryDataProto proto = ContainerHistoryDataProto.getDefaultInstance(); @@ -233,7 +230,6 @@ public void setFinalContainerStatus( convertToProtoFormat(finalContainerState)); } - @Override public ContainerHistoryDataProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index dc24707..b64d8ca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -47,6 +47,11 @@ test-jar test + + org.apache.hadoop + hadoop-yarn-server-applicationhistoryservice + 2.2.0-SNAPSHOT + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index b29da14..11d31a9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; @@ -43,6 +44,8 @@ RMStateStore getStateStore(); + RMApplicationHistoryWriter getApplicationHistoryWriter(); + ConcurrentMap getRMApps(); ConcurrentMap getInactiveRMNodes(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index a518911..d5521be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -34,8 +35,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; -import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import com.google.common.annotations.VisibleForTesting; @@ -55,6 +56,7 @@ private AMLivelinessMonitor amLivelinessMonitor; private AMLivelinessMonitor amFinishingMonitor; private RMStateStore stateStore = null; + private RMApplicationHistoryWriter ahWriter = null; private ContainerAllocationExpirer containerAllocationExpirer; private final DelegationTokenRenewer tokenRenewer; private final AMRMTokenSecretManager appTokenSecretManager; @@ -63,7 +65,7 @@ private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager; public RMContextImpl(Dispatcher rmDispatcher, - RMStateStore store, + RMStateStore store, RMApplicationHistoryWriter ahWriter, ContainerAllocationExpirer containerAllocationExpirer, AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor amFinishingMonitor, @@ -74,6 +76,7 @@ public RMContextImpl(Dispatcher rmDispatcher, ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager) { this.rmDispatcher = rmDispatcher; this.stateStore = store; + this.ahWriter = ahWriter; this.containerAllocationExpirer = containerAllocationExpirer; this.amLivelinessMonitor = amLivelinessMonitor; this.amFinishingMonitor = amFinishingMonitor; @@ -87,6 +90,7 @@ public RMContextImpl(Dispatcher rmDispatcher, @VisibleForTesting // helper constructor for tests public RMContextImpl(Dispatcher rmDispatcher, + RMApplicationHistoryWriter ahWriter, ContainerAllocationExpirer containerAllocationExpirer, AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor amFinishingMonitor, @@ -95,10 +99,10 @@ public RMContextImpl(Dispatcher rmDispatcher, RMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInRM nmTokenSecretManager, ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager) { - this(rmDispatcher, null, containerAllocationExpirer, amLivelinessMonitor, - amFinishingMonitor, tokenRenewer, appTokenSecretManager, - containerTokenSecretManager, nmTokenSecretManager, - clientToAMTokenSecretManager); + this(rmDispatcher, null, ahWriter, containerAllocationExpirer, + amLivelinessMonitor, amFinishingMonitor, tokenRenewer, + appTokenSecretManager, containerTokenSecretManager, + nmTokenSecretManager, clientToAMTokenSecretManager); RMStateStore nullStore = new NullRMStateStore(); nullStore.setDispatcher(rmDispatcher); try { @@ -120,6 +124,11 @@ public RMStateStore getStateStore() { } @Override + public RMApplicationHistoryWriter getApplicationHistoryWriter() { + return ahWriter; + } + + @Override public ConcurrentMap getRMApps() { return this.applications; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 841f387..b52c4b5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy; @@ -195,9 +196,18 @@ protected void serviceInit(Configuration conf) throws Exception { LOG.error("Failed to init state store", e); ExitUtil.terminate(1, e); } - + + RMApplicationHistoryWriter ahWriter = + new RMApplicationHistoryWriter(); + try { + ahWriter.init(conf); + } catch (Exception e) { + LOG.error("Failed to init RMApplicationHistoryWriter", e); + ExitUtil.terminate(1, e); + } + this.rmContext = - new RMContextImpl(this.rmDispatcher, rmStore, + new RMContextImpl(this.rmDispatcher, rmStore, ahWriter, this.containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, tokenRenewer, this.amRmTokenSecretManager, this.containerTokenSecretManager, this.nmTokenSecretManager, @@ -681,6 +691,14 @@ protected void serviceStop() throws Exception { } catch (Exception e) { LOG.error("Error closing store.", e); } + + RMApplicationHistoryWriter ahWriter = + rmContext.getApplicationHistoryWriter(); + try { + ahWriter.close(); + } catch (Exception e) { + LOG.error("Error when closing RMApplicationHistoryWriter.", e); + } } super.serviceStop(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java new file mode 100644 index 0000000..edaa99d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.ahs; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.applicationhistoryservice.DummyApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationHistoryDataPBImpl; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; + +/** + *

+ * {@link ResourceManager} uses this class to write {@link RMApp}, + * {@link RMAppAttempt} and {@link RMContainer}. These APIs are non-blocking, + * and just schedule a writing history event. An self-contained dispatcher will + * handle the event in a separate thread, and extract the exact fields that are + * going to be persisted. Then, the extracted information will be persisted via + * the implementation of {@link ApplicationHistoryWriter}. + *

+ */ +@Private +@Unstable +public class RMApplicationHistoryWriter extends AbstractService { + + public static final Log LOG = + LogFactory.getLog(RMApplicationHistoryWriter.class); + + protected AsyncDispatcher dispatcher; + protected ApplicationHistoryWriter writer; + + public RMApplicationHistoryWriter() { + super(RMApplicationHistoryWriter.class.getName()); + } + + @Override + protected synchronized void serviceInit( + Configuration conf) throws Exception { + dispatcher = new AsyncDispatcher(); + dispatcher.init(conf); + dispatcher.register( + WritingHistoryEventType.class, new ForwardingEventHandler()); + dispatcher.start(); + + boolean ahWriterEnabled = conf.getBoolean( + YarnConfiguration.RM_AHS_WRITER_ENABLED, + YarnConfiguration.DEFAULT_RM_AHS_WRITER_ENABLED); + if (ahWriterEnabled) { + try { + Class writerClass = + conf.getClass(YarnConfiguration.RM_AHS_WRITER_CLASS, + DummyApplicationHistoryWriter.class, + ApplicationHistoryWriter.class); + writer = (ApplicationHistoryWriter) writerClass.newInstance(); + if (writer instanceof Service) { + ((Service) writer).init(conf); + ((Service) writer).start(); + } + } catch (Exception e) { + LOG.error("Could not instantiate ApplicationHistoryWriter: " + + conf.get(YarnConfiguration.RM_AHS_WRITER_CLASS, + DummyApplicationHistoryWriter.class.getName()), e); + throw e; + } + } else { + writer = new DummyApplicationHistoryWriter(); + } + + super.serviceInit(conf); + } + + @Override + protected synchronized void serviceStop() throws Exception { + dispatcher.stop(); + if (writer instanceof Service) { + ((Service) writer).stop(); + } + super.serviceStop(); + } + + /** + * Non-blocking API + * {@link ResourceManager} should call this method to record the finished + * {@link RMApp}. + * @param app the {@link RMApp} object to be recorded + */ + @SuppressWarnings("unchecked") + public synchronized void writeApplication(RMApp app) { + assert app.getApplicationId() instanceof ApplicationIdPBImpl; + ApplicationHistoryData appHD = ApplicationHistoryDataPBImpl.newInstance( + app.getApplicationId(), app.getName(), app.getApplicationType(), + app.getQueue(), app.getUser(), app.getSubmitTime(), app.getStartTime(), + app.getFinishTime(), app.getDiagnostics().toString(), + app.getFinalApplicationStatus()); + dispatcher.getEventHandler().handle(new WritingAppHistoryEvent(appHD)); + } + + /** + * Non-blocking API + * {@link ResourceManager} should call this method to record the finished + * {@link RMAppAttempt}. + * @param appAttempt the {@link RMAppAttempt} object to be recorded + */ + @SuppressWarnings("unchecked") + public synchronized void writeApplicationAttempt(RMAppAttempt appAttempt) { + assert appAttempt.getAppAttemptId() instanceof ApplicationAttemptIdPBImpl; + assert appAttempt.getMasterContainer().getId() instanceof ContainerIdPBImpl; + ApplicationAttemptHistoryData appAttemptHD = ApplicationAttemptHistoryData + .newInstance(appAttempt.getAppAttemptId(), appAttempt.getHost(), + appAttempt.getRpcPort(), appAttempt.getMasterContainer().getId(), + appAttempt.getDiagnostics(), appAttempt.getTrackingUrl(), + appAttempt.getFinalApplicationStatus()); + dispatcher.getEventHandler().handle( + new WritingAppAttemptHistoryEvent(appAttemptHD)); + } + + /** + * Non-blocking API + * {@link ResourceManager} should call this method to record the finished + * {@link RMContainer}. + * @param container the {@link RMContainer} object to be recorded + */ + @SuppressWarnings("unchecked") + public synchronized void writeContainer(RMContainer container) { + // TODO: RMContainer needs to record more information to fill the blank + // fields of ContainerIdPBImpl bellow + assert container.getContainerId() instanceof ContainerIdPBImpl; + ContainerHistoryData containerHD = ContainerHistoryData.newInstance( + container.getContainerId(), null, null, null, 0, 0, null, null, + ContainerState.COMPLETE); + dispatcher.getEventHandler().handle( + new WritingContainerHistoryEvent(containerHD)); + } + + protected synchronized void handleWritingHistoryEvent( + WritingHistoryEvent event) { + switch(event.getType()) { + case WRITING_APP: + WritingAppHistoryEvent appEvent = (WritingAppHistoryEvent) event; + ApplicationHistoryData appHD = appEvent.getApplication(); + try { + LOG.info("Writing the history data for Application " + + appHD.getApplicationId()); + writer.writeApplication(appHD); + } catch (Throwable t) { + LOG.error("Failed to write the history data for Application " + + appHD.getApplicationId()); + } + break; + case WRITING_APP_ATTEMPT: + WritingAppAttemptHistoryEvent appAttemptEvent = + (WritingAppAttemptHistoryEvent) event; + ApplicationAttemptHistoryData appAttemptHD = + appAttemptEvent.getApplicationAttempt(); + try { + LOG.info("Writing the history data for ApplicationAttempt " + + appAttemptHD.getApplicationAttemptId()); + writer.writeApplicationAttempt(appAttemptHD); + } catch (Throwable t) { + LOG.error("Failed to write the history data for ApplicationAttempt " + + appAttemptHD.getApplicationAttemptId()); + } + break; + case WRITING_CONTAINER: + WritingContainerHistoryEvent containerEvent = + (WritingContainerHistoryEvent) event; + ContainerHistoryData containerHD = containerEvent.getContainer(); + try { + LOG.info("Writing the history data for Container " + + containerHD.getContainerId()); + writer.writeContainer(containerHD); + } catch (Throwable t) { + LOG.error("Failed to write the history data for Container " + + containerHD.getContainerId()); + } + break; + default: + LOG.error("Unknown WritingHistoryEvent type: " + event.getType()); + } + } + + /** + * EventHandler implementation which forward events to HistoryWriter + * Making use of it, HistoryWriter can avoid to have a public handle method + */ + protected final class ForwardingEventHandler + implements EventHandler { + + @Override + public void handle(WritingHistoryEvent event) { + handleWritingHistoryEvent(event); + } + + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingAppAttemptHistoryEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingAppAttemptHistoryEvent.java new file mode 100644 index 0000000..f2e925f --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingAppAttemptHistoryEvent.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.ahs; + +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; + + +public class WritingAppAttemptHistoryEvent extends WritingHistoryEvent { + + private ApplicationAttemptHistoryData appAttempt; + + public WritingAppAttemptHistoryEvent( + ApplicationAttemptHistoryData appAttempt) { + super(WritingHistoryEventType.WRITING_APP_ATTEMPT); + this.appAttempt = appAttempt; + } + + public ApplicationAttemptHistoryData getApplicationAttempt() { + return appAttempt; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingAppHistoryEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingAppHistoryEvent.java new file mode 100644 index 0000000..694e09f --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingAppHistoryEvent.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.ahs; + +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; + + +public class WritingAppHistoryEvent extends WritingHistoryEvent { + + private ApplicationHistoryData app; + + public WritingAppHistoryEvent(ApplicationHistoryData app) { + super(WritingHistoryEventType.WRITING_APP); + this.app = app; + } + + public ApplicationHistoryData getApplication() { + return app; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingContainerHistoryEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingContainerHistoryEvent.java new file mode 100644 index 0000000..47dfca4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingContainerHistoryEvent.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.yarn.server.resourcemanager.ahs; + +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; + + +public class WritingContainerHistoryEvent extends WritingHistoryEvent { + + private ContainerHistoryData container; + + public WritingContainerHistoryEvent(ContainerHistoryData container) { + super(WritingHistoryEventType.WRITING_CONTAINER); + this.container = container; + } + + public ContainerHistoryData getContainer() { + return container; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingHistoryEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingHistoryEvent.java new file mode 100644 index 0000000..46363f3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingHistoryEvent.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.ahs; + +import org.apache.hadoop.yarn.event.AbstractEvent; + + +public class WritingHistoryEvent extends AbstractEvent { + + public WritingHistoryEvent(WritingHistoryEventType type) { + super(type); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingHistoryEventType.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingHistoryEventType.java new file mode 100644 index 0000000..5070599 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/WritingHistoryEventType.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.ahs; + + +public enum WritingHistoryEventType { + WRITING_APP, + WRITING_APP_ATTEMPT, + WRITING_CONTAINER +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index f2ac3f5..7f96b11 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -716,6 +716,7 @@ public void transition(RMAppImpl app, RMAppEvent event) { app.handler.handle( new RMAppManagerEvent(app.applicationId, RMAppManagerEventType.APP_COMPLETED)); + app.rmContext.getApplicationHistoryWriter().writeApplication(app); }; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 11fdd94..11a7a22 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -778,6 +778,9 @@ public void transition(RMAppAttemptImpl appAttempt, rejectedEvent.getApplicationAttemptId().getApplicationId(), message) ); + + appAttempt.rmContext.getApplicationHistoryWriter() + .writeApplicationAttempt(appAttempt); } } @@ -910,6 +913,9 @@ public void transition(RMAppAttemptImpl appAttempt, // Remove the AppAttempt from the AMRMTokenSecretManager appAttempt.rmContext.getAMRMTokenSecretManager() .applicationMasterFinished(appAttemptId); + + appAttempt.rmContext.getApplicationHistoryWriter() + .writeApplicationAttempt(appAttempt); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index d44fd3f..7a747b0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAcquiredEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; @@ -133,6 +134,7 @@ private final ApplicationAttemptId appAttemptId; private final NodeId nodeId; private final Container container; + private final RMContext rmContext; private final EventHandler eventHandler; private final ContainerAllocationExpirer containerAllocationExpirer; @@ -142,15 +144,15 @@ public RMContainerImpl(Container container, ApplicationAttemptId appAttemptId, NodeId nodeId, - EventHandler handler, - ContainerAllocationExpirer containerAllocationExpirer) { + RMContext rmContext) { this.stateMachine = stateMachineFactory.make(this); this.containerId = container.getId(); this.nodeId = nodeId; this.container = container; this.appAttemptId = appAttemptId; - this.eventHandler = handler; - this.containerAllocationExpirer = containerAllocationExpirer; + this.rmContext = rmContext; + this.eventHandler = rmContext.getDispatcher().getEventHandler(); + this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer(); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); @@ -291,6 +293,9 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { // Inform AppAttempt container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( container.appAttemptId, finishedEvent.getRemoteContainerStatus())); + + container.rmContext.getApplicationHistoryWriter() + .writeContainer(container); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index a261dbf..1ec38a0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -276,9 +276,7 @@ synchronized public RMContainer allocate(NodeType type, FiCaSchedulerNode node, // Create RMContainer RMContainer rmContainer = new RMContainerImpl(container, this - .getApplicationAttemptId(), node.getNodeID(), this.rmContext - .getDispatcher().getEventHandler(), this.rmContext - .getContainerAllocationExpirer()); + .getApplicationAttemptId(), node.getNodeID(), rmContext); // Add it to allContainers list. newlyAllocatedContainers.add(rmContainer); @@ -401,8 +399,7 @@ public synchronized RMContainer reserve(FiCaSchedulerNode node, Priority priorit if (rmContainer == null) { rmContainer = new RMContainerImpl(container, getApplicationAttemptId(), - node.getNodeID(), rmContext.getDispatcher().getEventHandler(), - rmContext.getContainerAllocationExpirer()); + node.getNodeID(), rmContext); Resources.addTo(currentReservation, container.getResource()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index 670e961..5b416c4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -326,8 +326,7 @@ public synchronized RMContainer reserve(FSSchedulerNode node, Priority priority, if (rmContainer == null) { rmContainer = new RMContainerImpl(container, getApplicationAttemptId(), - node.getNodeID(), rmContext.getDispatcher().getEventHandler(), - rmContext.getContainerAllocationExpirer()); + node.getNodeID(), rmContext); Resources.addTo(currentReservation, container.getResource()); @@ -539,9 +538,7 @@ else if (allowed.equals(NodeType.RACK_LOCAL) && // Create RMContainer RMContainer rmContainer = new RMContainerImpl(container, - getApplicationAttemptId(), node.getNodeID(), rmContext - .getDispatcher().getEventHandler(), rmContext - .getContainerAllocationExpirer()); + getApplicationAttemptId(), node.getNodeID(), rmContext); // Add it to allContainers list. newlyAllocatedContainers.add(rmContainer); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 6698412..7be9238 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -99,7 +99,7 @@ public static RMContext mockRMContext(int n, long time) { rmDispatcher); AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor( rmDispatcher); - return new RMContextImpl(rmDispatcher, + return new RMContextImpl(rmDispatcher, null, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, null, null, null, null) { @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index fbec326..9dad521 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -83,7 +83,7 @@ public void setUp() throws Exception { InlineDispatcher rmDispatcher = new InlineDispatcher(); rmContext = - new RMContextImpl(rmDispatcher, null, null, null, + new RMContextImpl(rmDispatcher, null, null, null, null, mock(DelegationTokenRenewer.class), null, null, null, null); scheduler = mock(YarnScheduler.class); doAnswer( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java new file mode 100644 index 0000000..beb689c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.ahs; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.concurrent.atomic.AtomicBoolean; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.DrainDispatcher; +import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + + +public class TestRMApplicationHistoryWriter { + + private RMApplicationHistoryWriter rmWriter; + private MyApplicationHistoryWriter writer; + private DrainDispatcher dispatcher; + + @Before + public void setup() { + YarnConfiguration conf = new YarnConfiguration(); + rmWriter = new RMApplicationHistoryWriter() { + @Override + protected void serviceInit(Configuration conf) throws Exception { + super.serviceInit(conf); + + dispatcher = new DrainDispatcher(); + dispatcher.init(conf); + dispatcher.register( + WritingHistoryEventType.class, new ForwardingEventHandler()); + dispatcher.start(); + + writer = new MyApplicationHistoryWriter(); + } + }; + rmWriter.init(conf); + writer = (MyApplicationHistoryWriter) rmWriter.writer; + dispatcher = (DrainDispatcher) rmWriter.dispatcher; + } + + @After + public void tearDown() { + rmWriter.stop(); + } + + @Test + public void testWriteApplication() { + RMApp app = + new MockRMApp(1, System.currentTimeMillis(), RMAppState.RUNNING); + rmWriter.writeApplication(app); + dispatcher.await(); + Assert.assertTrue(writer.appWritten.get()); + } + + @Test + public void testWriteApplicationAttempt() { + RMAppAttempt appAttempt = mock(RMAppAttempt.class); + when(appAttempt.getAppAttemptId()).thenReturn( + ApplicationAttemptId.newInstance( + ApplicationId.newInstance(System.currentTimeMillis(), 1), 1)); + when(appAttempt.getHost()).thenReturn("localhost"); + when(appAttempt.getRpcPort()).thenReturn(9000); + Container container = mock(Container.class); + when(container.getId()).thenReturn( + ContainerId.newInstance(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1)); + when(appAttempt.getMasterContainer()).thenReturn(container); + when(appAttempt.getDiagnostics()).thenReturn("N/A"); + when(appAttempt.getTrackingUrl()).thenReturn("http://localhost/log"); + when(appAttempt.getFinalApplicationStatus()).thenReturn( + FinalApplicationStatus.SUCCEEDED); + rmWriter.writeApplicationAttempt(appAttempt); + dispatcher.await(); + Assert.assertTrue(writer.appAttemptWritten.get()); + } + + @Test + public void testWriteContainer() { + RMContainer container = mock(RMContainer.class); + when(container.getContainerId()).thenReturn( + ContainerId.newInstance(ApplicationAttemptId.newInstance( + ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1)); + rmWriter.writeContainer(container); + dispatcher.await(); + Assert.assertTrue(writer.containerWritten.get()); + } + + private static class MyApplicationHistoryWriter + implements ApplicationHistoryWriter { + + private AtomicBoolean appWritten = new AtomicBoolean(false); + private AtomicBoolean appAttemptWritten = new AtomicBoolean(false); + private AtomicBoolean containerWritten = new AtomicBoolean(false); + + @Override + public void writeApplication(ApplicationHistoryData app) + throws Throwable { + appWritten.set(true); + } + + @Override + public void writeApplicationAttempt( + ApplicationAttemptHistoryData appAttempt) throws Throwable { + appAttemptWritten.set(true); + } + + @Override + public void writeContainer(ContainerHistoryData container) + throws Throwable { + containerWritten.set(true); + } + + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index a884552..d375fce 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -71,7 +71,7 @@ public void setUp() { Configuration conf = new Configuration(); // Dispatcher that processes events inline Dispatcher dispatcher = new InlineDispatcher(); - RMContext context = new RMContextImpl(dispatcher, null, + RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, null); dispatcher.register(SchedulerEventType.class, new InlineDispatcher.EmptyEventHandler()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java index ddb7a90..361cc8c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java @@ -69,7 +69,7 @@ public void handle(Event event) { } }); RMContext context = - new RMContextImpl(dispatcher, null, null, null, null, + new RMContextImpl(dispatcher, null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), null); dispatcher.register(RMNodeEventType.class, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index a4b7404..2d523b8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -18,7 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.io.IOException; @@ -41,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -67,6 +71,7 @@ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS; private static int appId = 1; private DrainDispatcher rmDispatcher; + private RMApplicationHistoryWriter ahWriter; // ignore all the RM application attempt events private static final class TestApplicationAttemptEventDispatcher implements @@ -142,8 +147,9 @@ public void setUp() throws Exception { AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class); AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class); RMStateStore store = mock(RMStateStore.class); + ahWriter = mock(RMApplicationHistoryWriter.class); this.rmContext = - new RMContextImpl(rmDispatcher, store, + new RMContextImpl(rmDispatcher, store, ahWriter, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, new AMRMTokenSecretManager(conf), new RMContainerTokenSecretManager(conf), @@ -370,6 +376,7 @@ protected RMApp testCreateAppFinished( assertFinalAppStatus(FinalApplicationStatus.FAILED, application); Assert.assertTrue("Finished app missing diagnostics", application.getDiagnostics().indexOf(diagnostics) != -1); + verify(ahWriter).writeApplication(any(RMApp.class)); return application; } @@ -384,6 +391,7 @@ public void testUnmanagedApp() throws IOException { RMApp application = testCreateAppFinished(subContext, diagMsg); Assert.assertTrue("Finished app missing diagnostics", application.getDiagnostics().indexOf(diagMsg) != -1); + verify(ahWriter).writeApplication(any(RMApp.class)); // test app fails after 1 app attempt failure LOG.info("--- START: testUnmanagedAppFailPath ---"); @@ -396,6 +404,7 @@ public void testUnmanagedApp() throws IOException { Assert.assertEquals(1, appAttempt.getAppAttemptId().getAttemptId()); assertFailed(application, ".*Unmanaged application.*Failing the application.*"); + verify(ahWriter, times(2)).writeApplication(any(RMApp.class)); } @Test @@ -405,6 +414,7 @@ public void testAppSuccessPath() throws IOException { RMApp application = testCreateAppFinished(null, diagMsg); Assert.assertTrue("Finished application missing diagnostics", application.getDiagnostics().indexOf(diagMsg) != -1); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test (timeout = 30000) @@ -424,6 +434,7 @@ public void testAppNewKill() throws IOException { application.handle(event); rmDispatcher.await(); assertKilled(application); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test @@ -438,6 +449,7 @@ public void testAppNewReject() throws IOException { application.handle(event); rmDispatcher.await(); assertFailed(application, rejectedText); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test (timeout = 30000) @@ -451,6 +463,7 @@ public void testAppNewSavingKill() throws IOException { application.handle(event); rmDispatcher.await(); assertKilled(application); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test (timeout = 30000) @@ -465,6 +478,7 @@ public void testAppNewSavingReject() throws IOException { application.handle(event); rmDispatcher.await(); assertFailed(application, rejectedText); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test (timeout = 30000) @@ -479,6 +493,7 @@ public void testAppSubmittedRejected() throws IOException { application.handle(event); rmDispatcher.await(); assertFailed(application, rejectedText); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test @@ -494,6 +509,7 @@ public void testAppSubmittedKill() throws IOException, InterruptedException { rmDispatcher.await(); assertKilled(application); assertAppAndAttemptKilled(application); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test @@ -526,6 +542,7 @@ public void testAppAcceptedFailed() throws IOException { application.handle(event); rmDispatcher.await(); assertFailed(application, ".*" + message + ".*Failing the application.*"); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test @@ -541,6 +558,7 @@ public void testAppAcceptedKill() throws IOException, InterruptedException { rmDispatcher.await(); assertKilled(application); assertAppAndAttemptKilled(application); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test @@ -554,6 +572,7 @@ public void testAppRunningKill() throws IOException { application.handle(event); rmDispatcher.await(); assertKilled(application); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test @@ -599,6 +618,7 @@ public void testAppRunningFailed() throws IOException { application.handle(event); rmDispatcher.await(); assertFailed(application, ".*Failing the application.*"); + verify(ahWriter).writeApplication(any(RMApp.class)); // FAILED => FAILED event RMAppEventType.KILL event = new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL); @@ -618,6 +638,7 @@ public void testAppFinishingKill() throws IOException { application.handle(event); rmDispatcher.await(); assertAppState(RMAppState.FINISHED, application); + verify(ahWriter).writeApplication(any(RMApp.class)); } @Test @@ -650,6 +671,7 @@ public void testAppFailedFailed() throws IOException { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); + verify(ahWriter).writeApplication(any(RMApp.class)); // FAILED => FAILED event RMAppEventType.KILL event = @@ -680,6 +702,7 @@ public void testAppKilledKilled() throws IOException { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); + verify(ahWriter).writeApplication(any(RMApp.class)); // KILLED => KILLED event RMAppEventType.ATTEMPT_FINISHED event = new RMAppFinishedAttemptEvent( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index cafe4f9..f8ccb9d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; @@ -94,6 +95,7 @@ YarnConfiguration.getRMWebAppHostAndPort(new Configuration()); private RMContext rmContext; + private RMApplicationHistoryWriter ahWriter; private YarnScheduler scheduler; private ApplicationMasterService masterService; private ApplicationMasterLauncher applicationMasterLauncher; @@ -164,8 +166,9 @@ public void setUp() throws Exception { amLivelinessMonitor = mock(AMLivelinessMonitor.class); amFinishingMonitor = mock(AMLivelinessMonitor.class); Configuration conf = new Configuration(); + ahWriter = mock(RMApplicationHistoryWriter.class); rmContext = - new RMContextImpl(rmDispatcher, + new RMContextImpl(rmDispatcher, ahWriter, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, new AMRMTokenSecretManager(conf), new RMContainerTokenSecretManager(conf), @@ -303,6 +306,7 @@ private void testAppAttemptKilledState(Container amContainer, assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001); assertEquals(0, applicationAttempt.getRanNodes().size()); assertNull(applicationAttempt.getFinalApplicationStatus()); + verify(ahWriter).writeApplicationAttempt(any(RMAppAttempt.class)); } /** @@ -377,6 +381,8 @@ private void testAppAttemptFailedState(Container container, // Check events verify(application, times(2)).handle(any(RMAppFailedAttemptEvent.class)); + + verify(ahWriter).writeApplicationAttempt(any(RMAppAttempt.class)); } /** @@ -442,6 +448,7 @@ private void testAppAttemptFinishedState(Container container, .getJustFinishedContainers().size()); assertEquals(container, applicationAttempt.getMasterContainer()); assertEquals(finalStatus, applicationAttempt.getFinalApplicationStatus()); + verify(ahWriter).writeApplicationAttempt(any(RMAppAttempt.class)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index d1262d8..0e7aa4d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -19,9 +19,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -34,6 +36,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; @@ -46,6 +50,7 @@ @SuppressWarnings({ "unchecked", "rawtypes" }) public class TestRMContainerImpl { + @SuppressWarnings("unused") @Test public void testReleaseWhileRunning() { @@ -71,8 +76,15 @@ public void testReleaseWhileRunning() { Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); + RMApplicationHistoryWriter ahWriter = + mock(RMApplicationHistoryWriter.class); + RMContext rmContext = mock(RMContext.class); + when(rmContext.getDispatcher()).thenReturn(drainDispatcher); + when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); + when(rmContext.getApplicationHistoryWriter()).thenReturn(ahWriter); + RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, - nodeId, eventHandler, expirer); + nodeId, rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); @@ -114,8 +126,10 @@ public void testReleaseWhileRunning() { .createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED)); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); + verify(ahWriter).writeContainer(any(RMContainer.class)); } + @SuppressWarnings("unused") @Test public void testExpireWhileRunning() { @@ -141,8 +155,15 @@ public void testExpireWhileRunning() { Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); + RMApplicationHistoryWriter ahWriter = + mock(RMApplicationHistoryWriter.class); + RMContext rmContext = mock(RMContext.class); + when(rmContext.getDispatcher()).thenReturn(drainDispatcher); + when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); + when(rmContext.getApplicationHistoryWriter()).thenReturn(ahWriter); + RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, - nodeId, eventHandler, expirer); + nodeId, rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); @@ -170,5 +191,10 @@ public void testExpireWhileRunning() { containerStatus, RMContainerEventType.EXPIRE)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); + rmContainer.handle(new RMContainerFinishedEvent(containerId, + containerStatus, RMContainerEventType.KILL)); + drainDispatcher.await(); + assertEquals(RMContainerState.KILLED, rmContainer.getState()); + verify(ahWriter).writeContainer(any(RMContainer.class)); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index ec486d7..02f6825 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -31,7 +31,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -308,7 +307,7 @@ public void testRefreshQueues() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM())); @@ -407,7 +406,7 @@ public void testParseQueue() throws IOException { conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM())); @@ -420,7 +419,7 @@ public void testReconnectedNode() throws Exception { setupQueueConfiguration(csConf); CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); - cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, + cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM())); @@ -447,7 +446,7 @@ public void testRefreshQueuesWithNewQueue() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM())); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index c86d6b3..47a9c05 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -44,7 +44,7 @@ public void testQueueParsing() throws Exception { CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(conf); - capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, + capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM())); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 6e7fe78..336c357 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -85,7 +85,7 @@ public EventHandler getEventHandler() { Configuration conf = new Configuration(); RMContext rmContext = - new RMContextImpl(nullDispatcher, cae, null, null, null, + new RMContextImpl(nullDispatcher, null, cae, null, null, null, new AMRMTokenSecretManager(conf), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index b71726a..f535c79 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -126,7 +126,7 @@ public void testFifoSchedulerCapacityWhenNoNMs() { public void testAppAttemptMetrics() throws Exception { AsyncDispatcher dispatcher = new InlineDispatcher(); RMContext rmContext = new RMContextImpl(dispatcher, null, - null, null, null, null, null, null, null); + null, null, null, null, null, null, null, null); FifoScheduler schedular = new FifoScheduler(); schedular.reinitialize(new Configuration(), rmContext); @@ -159,7 +159,7 @@ public void testNodeLocalAssignment() throws Exception { new NMTokenSecretManagerInRM(conf); nmTokenSecretManager.rollMasterKey(); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, - null, containerTokenSecretManager, nmTokenSecretManager, null); + null, null, containerTokenSecretManager, nmTokenSecretManager, null); FifoScheduler scheduler = new FifoScheduler(); scheduler.reinitialize(new Configuration(), rmContext); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index aa2d6c6..a50e42f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -160,7 +160,7 @@ public static RMContext mockRMContext(int numApps, int racks, int numNodes, for (RMNode node : deactivatedNodes) { deactivatedNodesMap.put(node.getHostName(), node); } - return new RMContextImpl(null, null, null, null, + return new RMContextImpl(null, null, null, null, null, null, null, null, null, null) { @Override public ConcurrentMap getRMApps() { @@ -201,7 +201,7 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM()));