diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMBlackListingRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMBlackListingRequest.java deleted file mode 100644 index 4aec2ba..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMBlackListingRequest.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.util.Records; - -/** - * Specific AMBlacklistingRequest from AM to enable/disable blacklisting. - */ -@Public -@Evolving -public abstract class AMBlackListingRequest { - - @Private - @Unstable - public static AMBlackListingRequest newInstance( - boolean isAMBlackListingEnabled, float disableFailureThreshold) { - AMBlackListingRequest blackListRequest = Records - .newRecord(AMBlackListingRequest.class); - blackListRequest.setBlackListingEnabled(isAMBlackListingEnabled); - blackListRequest - .setBlackListingDisableFailureThreshold(disableFailureThreshold); - return blackListRequest; - } - - /** - * @return AM Blacklisting is enabled. - */ - @Public - @Evolving - public abstract boolean isAMBlackListingEnabled(); - - /** - * @return AM Blacklisting disable failure threshold - */ - @Public - @Evolving - public abstract float getBlackListingDisableFailureThreshold(); - - @Private - @Unstable - public abstract void setBlackListingEnabled(boolean isAMBlackListingEnabled); - - @Private - @Unstable - public abstract void setBlackListingDisableFailureThreshold( - float disableFailureThreshold); -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index fe833f7..21cd1bb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -535,27 +535,4 @@ public abstract void setLogAggregationContext( @Public @Unstable public abstract void setReservationID(ReservationId reservationID); - - /** - * Get AM Blacklisting request object to know whether application needs any - * specific blacklisting for AM Nodes. - * - * @return AMBlackListingRequest object which has blacklisting information. - */ - @Public - @Unstable - public abstract AMBlackListingRequest getAMBlackListRequest(); - - /** - * Get AM Blacklisting request object to know whether application needs any - * specific blacklisting for AM Nodes. - * - * @param blackListRequest - * object which has blacklisting information such as - * "enable/disable AM blacklisting" and "disable failure threshold". - */ - @Public - @Unstable - public abstract void setAMBlackListRequest( - AMBlackListingRequest blackListRequest); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0b150c2..57cf8b8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2524,14 +2524,25 @@ public static boolean areNodeLabelsEnabled( public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS = 30 * 60 * 1000; - public static final String AM_BLACKLISTING_ENABLED = - YARN_PREFIX + "am.blacklisting.enabled"; - public static final boolean DEFAULT_AM_BLACKLISTING_ENABLED = true; - - public static final String AM_BLACKLISTING_DISABLE_THRESHOLD = - YARN_PREFIX + "am.blacklisting.disable-failure-threshold"; - public static final float DEFAULT_AM_BLACKLISTING_DISABLE_THRESHOLD = 0.8f; + @Private + /** + * This is a private feature that isn't supposed to be used by end-users. + */ + public static final String AM_SCHEDULING_NODE_BLACKLISTING_ENABLED = + RM_PREFIX + "am-scheduling.node-blacklisting-enabled"; + @Private + public static final boolean DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_ENABLED = + true; + @Private + /** + * This is a private feature that isn't supposed to be used by end-users. + */ + public static final String AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD = + RM_PREFIX + "am-scheduling.node-blacklisting-disable-threshold"; + @Private + public static final float DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD = + 0.8f; private static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX = NM_NODE_LABELS_PROVIDER_PREFIX + "script."; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 60cdfd1..a67e8c0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -290,10 +290,6 @@ enum ExecutionTypeProto { OPPORTUNISTIC = 2; } -message AMBlackListingRequestProto { - optional bool blacklisting_enabled = 1 [default = false]; - optional float blacklisting_failure_threshold = 2; -} //////////////////////////////////////////////////////////////////////// ////// From AM_RM_Protocol ///////////////////////////////////////////// //////////////////////////////////////////////////////////////////////// @@ -360,7 +356,6 @@ message ApplicationSubmissionContextProto { optional ReservationIdProto reservation_id = 15; optional string node_label_expression = 16; optional ResourceRequestProto am_container_resource_request = 17; - optional AMBlackListingRequestProto am_blacklisting_request = 18; } message LogAggregationContextProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMBlackListingRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMBlackListingRequestPBImpl.java deleted file mode 100644 index 1d04dd2..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMBlackListingRequestPBImpl.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records.impl.pb; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.AMBlackListingRequest; -import org.apache.hadoop.yarn.proto.YarnProtos.AMBlackListingRequestProto; -import org.apache.hadoop.yarn.proto.YarnProtos.AMBlackListingRequestProtoOrBuilder; - -import com.google.protobuf.TextFormat; - -@Private -@Unstable -public class AMBlackListingRequestPBImpl extends AMBlackListingRequest { - AMBlackListingRequestProto proto = AMBlackListingRequestProto - .getDefaultInstance(); - AMBlackListingRequestProto.Builder builder = null; - boolean viaProto = false; - - public AMBlackListingRequestPBImpl() { - builder = AMBlackListingRequestProto.newBuilder(); - } - - public AMBlackListingRequestPBImpl(AMBlackListingRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public AMBlackListingRequestProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = AMBlackListingRequestProto.newBuilder(proto); - } - viaProto = false; - } - - @Override - public boolean isAMBlackListingEnabled() { - AMBlackListingRequestProtoOrBuilder p = viaProto ? proto : builder; - return p.getBlacklistingEnabled(); - } - - @Override - public float getBlackListingDisableFailureThreshold() { - AMBlackListingRequestProtoOrBuilder p = viaProto ? proto : builder; - return p.getBlacklistingFailureThreshold(); - } - - @Override - public void setBlackListingEnabled(boolean isAMBlackListingEnabled) { - maybeInitBuilder(); - builder.setBlacklistingEnabled(isAMBlackListingEnabled); - } - - @Override - public void setBlackListingDisableFailureThreshold( - float disableFailureThreshold) { - maybeInitBuilder(); - builder.setBlacklistingFailureThreshold(disableFailureThreshold); - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) { - return false; - } - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return TextFormat.shortDebugString(getProto()); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index b39258e..67e3a84 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -24,7 +24,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.records.AMBlackListingRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -34,7 +33,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.proto.YarnProtos.AMBlackListingRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder; @@ -65,7 +63,6 @@ private ResourceRequest amResourceRequest = null; private LogAggregationContext logAggregationContext = null; private ReservationId reservationId = null; - private AMBlackListingRequest amBlackListRequest = null; public ApplicationSubmissionContextPBImpl() { builder = ApplicationSubmissionContextProto.newBuilder(); @@ -134,10 +131,6 @@ private void mergeLocalToBuilder() { if (this.reservationId != null) { builder.setReservationId(convertToProtoFormat(this.reservationId)); } - if (this.amBlackListRequest != null) { - builder.setAmBlacklistingRequest( - convertToProtoFormat(this.amBlackListRequest)); - } } private void mergeLocalToProto() { @@ -420,29 +413,6 @@ public boolean getKeepContainersAcrossApplicationAttempts() { return p.getKeepContainersAcrossApplicationAttempts(); } - @Override - public AMBlackListingRequest getAMBlackListRequest() { - ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; - if (amBlackListRequest != null) { - return amBlackListRequest; - } - if (!p.hasAmBlacklistingRequest()) { - return null; - } - amBlackListRequest = convertFromProtoFormat(p.getAmBlacklistingRequest()); - return amBlackListRequest; - } - - @Override - public void setAMBlackListRequest(AMBlackListingRequest amBlackListRequest) { - maybeInitBuilder(); - if (amBlackListRequest == null) { - builder.clearAmBlacklistingRequest(); - return; - } - this.amBlackListRequest = amBlackListRequest; - } - private PriorityPBImpl convertFromProtoFormat(PriorityProto p) { return new PriorityPBImpl(p); } @@ -485,16 +455,6 @@ private ResourceProto convertToProtoFormat(Resource t) { return ((ResourcePBImpl)t).getProto(); } - private AMBlackListingRequestPBImpl convertFromProtoFormat( - AMBlackListingRequestProto a) { - return new AMBlackListingRequestPBImpl(a); - } - - private AMBlackListingRequestProto convertToProtoFormat( - AMBlackListingRequest a) { - return ((AMBlackListingRequestPBImpl) a).getProto(); - } - @Override public String getNodeLabelExpression() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a38d0d8..2273802 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2648,25 +2648,6 @@ 4096 - - - Enable/disable blacklisting of hosts for AM based on AM failures on those - hosts. - - yarn.am.blacklisting.enabled - true - - - - - Threshold of ratio number of NodeManager hosts that are allowed to be - blacklisted for AM. Beyond this ratio there is no blacklisting to avoid - danger of blacklisting the entire cluster. - - yarn.am.blacklisting.disable-failure-threshold - 0.8f - - Choose different implementation of node label's storage diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 14f61b7..58375f4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -109,7 +109,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; -import org.apache.hadoop.yarn.api.records.AMBlackListingRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -154,7 +153,6 @@ import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; -import org.apache.hadoop.yarn.api.records.impl.pb.AMBlackListingRequestPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptReportPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; @@ -189,7 +187,6 @@ import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.URLPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.YarnClusterMetricsPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.AMBlackListingRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; @@ -513,7 +510,6 @@ public static void setup() throws Exception { generateByNewInstance(ResourceAllocationRequest.class); generateByNewInstance(ReservationAllocationState.class); generateByNewInstance(ResourceUtilization.class); - generateByNewInstance(AMBlackListingRequest.class); } private class GetSetPair { @@ -1353,10 +1349,4 @@ public void testCheckForDecommissioningNodesResponsePBImpl() throws Exception { validatePBImplRecord(CheckForDecommissioningNodesResponsePBImpl.class, CheckForDecommissioningNodesResponseProto.class); } - - @Test - public void testAMBlackListingRequestPBImpl() throws Exception { - validatePBImplRecord(AMBlackListingRequestPBImpl.class, - AMBlackListingRequestProto.class); - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/BlacklistManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/BlacklistManager.java index f03b421..f343603 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/BlacklistManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/BlacklistManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.blacklist; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; /** * Tracks blacklists based on failures reported on nodes. @@ -33,14 +34,14 @@ void addNode(String node); /** - * Get {@link BlacklistUpdates} that indicate which nodes should be + * Get {@link ResourceBlacklistRequest} that indicate which nodes should be * added or to removed from the blacklist. - * @return {@link BlacklistUpdates} + * @return {@link ResourceBlacklistRequest} */ - BlacklistUpdates getBlacklistUpdates(); + ResourceBlacklistRequest getBlacklistUpdates(); /** - * Refresh the number of nodemanager hosts available for scheduling. + * Refresh the number of NodeManagers available for scheduling. * @param nodeHostCount is the number of node hosts. */ void refreshNodeHostCount(int nodeHostCount); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/BlacklistUpdates.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/BlacklistUpdates.java deleted file mode 100644 index c76dfb4..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/BlacklistUpdates.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.blacklist; - -import org.apache.hadoop.classification.InterfaceAudience.Private; - -import java.util.List; - -/** - * Class to track blacklist additions and removals. - */ -@Private -public class BlacklistUpdates { - - private List additions; - private List removals; - - public BlacklistUpdates(List additions, - List removals) { - this.additions = additions; - this.removals = removals; - } - - public List getAdditions() { - return additions; - } - - public List getRemovals() { - return removals; - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/DisabledBlacklistManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/DisabledBlacklistManager.java index f155b45..8bb308d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/DisabledBlacklistManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/DisabledBlacklistManager.java @@ -20,21 +20,23 @@ import java.util.ArrayList; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; + /** * A {@link BlacklistManager} that returns no blacklists. */ -public class DisabledBlacklistManager implements BlacklistManager{ +public class DisabledBlacklistManager implements BlacklistManager { private static final ArrayList EMPTY_LIST = new ArrayList(); - private BlacklistUpdates noBlacklist = - new BlacklistUpdates(EMPTY_LIST, EMPTY_LIST); + private ResourceBlacklistRequest noBlacklist = + ResourceBlacklistRequest.newInstance(EMPTY_LIST, EMPTY_LIST); @Override public void addNode(String node) { } @Override - public BlacklistUpdates getBlacklistUpdates() { + public ResourceBlacklistRequest getBlacklistUpdates() { return noBlacklist; } @@ -42,4 +44,4 @@ public BlacklistUpdates getBlacklistUpdates() { public void refreshNodeHostCount(int nodeHostCount) { // Do nothing } -} +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java index c7bd0f8..f10e885 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java @@ -18,14 +18,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.blacklist; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; + /** * Maintains a list of failed nodes and returns that as long as number of * blacklisted nodes is below a threshold percentage of total nodes. If more @@ -58,8 +59,8 @@ public void refreshNodeHostCount(int nodeHostCount) { } @Override - public BlacklistUpdates getBlacklistUpdates() { - BlacklistUpdates ret; + public ResourceBlacklistRequest getBlacklistUpdates() { + ResourceBlacklistRequest ret; List blacklist = new ArrayList<>(blacklistNodes); final int currentBlacklistSize = blacklist.size(); final double failureThreshold = this.blacklistDisableFailureThreshold * @@ -70,13 +71,15 @@ public BlacklistUpdates getBlacklistUpdates() { "failure threshold ratio " + blacklistDisableFailureThreshold + " out of total usable nodes " + numberOfNodeManagerHosts); } - ret = new BlacklistUpdates(blacklist, EMPTY_LIST); + ret = ResourceBlacklistRequest.newInstance(blacklist, EMPTY_LIST); } else { LOG.warn("Ignoring Blacklists, blacklist size " + currentBlacklistSize + " is more than failure threshold ratio " + blacklistDisableFailureThreshold + " out of total usable nodes " + numberOfNodeManagerHosts); - ret = new BlacklistUpdates(EMPTY_LIST, blacklist); + // TODO: After the threshold hits, we will keep sending a long list + // every time a new AM is to be scheduled. + ret = ResourceBlacklistRequest.newInstance(EMPTY_LIST, blacklist); } return ret; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 07d5a74..aaddce4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -391,8 +391,8 @@ stateMachine; private static final int DUMMY_APPLICATION_ATTEMPT_NUMBER = -1; - private static final float MINIMUM_THRESHOLD_VALUE = 0.0f; - private static final float MAXIMUM_THRESHOLD_VALUE = 1.0f; + private static final float MINIMUM_AM_BLACKLIST_THRESHOLD_VALUE = 0.0f; + private static final float MAXIMUM_AM_BLACKLIST_THRESHOLD_VALUE = 1.0f; public RMAppImpl(ApplicationId applicationId, RMContext rmContext, Configuration config, String name, String user, String queue, @@ -471,42 +471,22 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, YarnConfiguration.RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY, YarnConfiguration.DEFAULT_RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY); - // amBlacklistingEnabled can be configured globally and by each - // application. - // Case 1: If AMBlackListRequest is available in submission context, we - // will consider only app level request (RM level configuration will be - // skipped). - // Case 2: AMBlackListRequest is available in submission context and - // amBlacklisting is disabled. In this case, AM blacklisting wont be - // enabled for this app even if this feature is enabled in RM level. - // Case 3: AMBlackListRequest is not available through submission context. - // RM level AM black listing configuration will be considered. - if (null != submissionContext.getAMBlackListRequest()) { - amBlacklistingEnabled = submissionContext.getAMBlackListRequest() - .isAMBlackListingEnabled(); - blacklistDisableThreshold = 0.0f; - if (amBlacklistingEnabled) { - blacklistDisableThreshold = submissionContext.getAMBlackListRequest() - .getBlackListingDisableFailureThreshold(); - - // Verify whether blacklistDisableThreshold is valid. And for invalid - // threshold, reset to global level blacklistDisableThreshold - // configured. - if (blacklistDisableThreshold < MINIMUM_THRESHOLD_VALUE - || blacklistDisableThreshold > MAXIMUM_THRESHOLD_VALUE) { - blacklistDisableThreshold = conf.getFloat( - YarnConfiguration.AM_BLACKLISTING_DISABLE_THRESHOLD, - YarnConfiguration.DEFAULT_AM_BLACKLISTING_DISABLE_THRESHOLD); - } - } - } else { - amBlacklistingEnabled = conf.getBoolean( - YarnConfiguration.AM_BLACKLISTING_ENABLED, - YarnConfiguration.DEFAULT_AM_BLACKLISTING_ENABLED); - if (amBlacklistingEnabled) { - blacklistDisableThreshold = conf.getFloat( - YarnConfiguration.AM_BLACKLISTING_DISABLE_THRESHOLD, - YarnConfiguration.DEFAULT_AM_BLACKLISTING_DISABLE_THRESHOLD); + // amBlacklistingEnabled can be configured globally + // Just use the global values + amBlacklistingEnabled = + conf.getBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, + YarnConfiguration.DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_ENABLED); + if (amBlacklistingEnabled) { + blacklistDisableThreshold = + conf.getFloat(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD, + YarnConfiguration.DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD); + // Verify whether blacklistDisableThreshold is valid. And for invalid + // threshold, reset to global level blacklistDisableThreshold + // configured. + if (blacklistDisableThreshold < MINIMUM_AM_BLACKLIST_THRESHOLD_VALUE + || blacklistDisableThreshold > MAXIMUM_AM_BLACKLIST_THRESHOLD_VALUE) { + blacklistDisableThreshold = + YarnConfiguration.DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD; } } } @@ -877,15 +857,16 @@ private void createNewAttempt() { } private void createNewAttempt(ApplicationAttemptId appAttemptId) { - BlacklistManager currentAMBlacklist; + BlacklistManager currentAMBlacklistManager; if (currentAttempt != null) { - currentAMBlacklist = currentAttempt.getAMBlacklist(); + // Transfer over the blacklist from the previous app-attempt. + currentAMBlacklistManager = currentAttempt.getAMBlacklistManager(); } else { if (amBlacklistingEnabled) { - currentAMBlacklist = new SimpleBlacklistManager( + currentAMBlacklistManager = new SimpleBlacklistManager( scheduler.getNumClusterNodes(), blacklistDisableThreshold); } else { - currentAMBlacklist = new DisabledBlacklistManager(); + currentAMBlacklistManager = new DisabledBlacklistManager(); } } RMAppAttempt attempt = @@ -896,7 +877,7 @@ private void createNewAttempt(ApplicationAttemptId appAttemptId) { // hardware error and NM resync) + 1) equal to the max-attempt // limit. maxAppAttempts == (getNumFailedAppAttempts() + 1), amReq, - currentAMBlacklist); + currentAMBlacklistManager); attempts.put(appAttemptId, attempt); currentAttempt = attempt; } @@ -1825,16 +1806,6 @@ private void sendATSCreateEvent(RMApp app, long startTime) { rmContext.getSystemMetricsPublisher().appCreated(app, startTime); } - @VisibleForTesting - public boolean isAmBlacklistingEnabled() { - return amBlacklistingEnabled; - } - - @VisibleForTesting - public float getAmBlacklistingDisableThreshold() { - return blacklistDisableThreshold; - } - @Private @VisibleForTesting public int getNextAttemptId() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index f2e60d1..cfd91e9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -190,7 +190,7 @@ * Get the {@link BlacklistManager} that manages blacklists for AM failures * @return the {@link BlacklistManager} that tracks AM failures. */ - BlacklistManager getAMBlacklist(); + BlacklistManager getAMBlacklistManager(); /** * the start time of the application. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1e2a293..2c28514 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -73,7 +74,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; import org.apache.hadoop.yarn.server.resourcemanager.blacklist.BlacklistManager; -import org.apache.hadoop.yarn.server.resourcemanager.blacklist.BlacklistUpdates; import org.apache.hadoop.yarn.server.resourcemanager.blacklist.DisabledBlacklistManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; @@ -493,7 +493,7 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq, - BlacklistManager amBlacklist) { + BlacklistManager amBlacklistManager) { this.conf = conf; this.applicationAttemptId = appAttemptId; this.rmContext = rmContext; @@ -514,7 +514,7 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, new RMAppAttemptMetrics(applicationAttemptId, rmContext); this.amReq = amReq; - this.blacklistedNodesForAM = amBlacklist; + this.blacklistedNodesForAM = amBlacklistManager; } @Override @@ -1032,15 +1032,15 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, appAttempt.amReq.setResourceName(ResourceRequest.ANY); appAttempt.amReq.setRelaxLocality(true); - appAttempt.getAMBlacklist().refreshNodeHostCount( + appAttempt.getAMBlacklistManager().refreshNodeHostCount( appAttempt.scheduler.getNumClusterNodes()); - BlacklistUpdates amBlacklist = appAttempt.getAMBlacklist() - .getBlacklistUpdates(); + ResourceBlacklistRequest amBlacklist = + appAttempt.getAMBlacklistManager().getBlacklistUpdates(); if (LOG.isDebugEnabled()) { LOG.debug("Using blacklist for AM: additions(" + - amBlacklist.getAdditions() + ") and removals(" + - amBlacklist.getRemovals() + ")"); + amBlacklist.getBlacklistAdditions() + ") and removals(" + + amBlacklist.getBlacklistRemovals() + ")"); } // AM resource has been checked when submission Allocation amContainerAllocation = @@ -1048,8 +1048,8 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, appAttempt.applicationAttemptId, Collections.singletonList(appAttempt.amReq), EMPTY_CONTAINER_RELEASE_LIST, - amBlacklist.getAdditions(), - amBlacklist.getRemovals(), null, null); + amBlacklist.getBlacklistAdditions(), + amBlacklist.getBlacklistRemovals(), null, null); if (amContainerAllocation != null && amContainerAllocation.getContainers() != null) { assert (amContainerAllocation.getContainers().size() == 0); @@ -1482,9 +1482,35 @@ public boolean shouldCountTowardsMaxAttemptRetry() { } } - private boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { - return !(exitStatus == ContainerExitStatus.SUCCESS - || exitStatus == ContainerExitStatus.PREEMPTED); + private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { + switch (exitStatus) { + case ContainerExitStatus.PREEMPTED: + case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER: + case ContainerExitStatus.KILLED_BY_APPMASTER: + case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION: + case ContainerExitStatus.ABORTED: + // Neither the app's fault nor the system's fault. This happens by design, + // so no need for skipping nodes + return false; + case ContainerExitStatus.DISKS_FAILED: + // The node should already be marked unhealthy, no need for any explicit + // skipping of nodes + return false; + case ContainerExitStatus.KILLED_EXCEEDED_VMEM: + case ContainerExitStatus.KILLED_EXCEEDED_PMEM: + // No point in skipping the node as it's not the system's fault + return false; + case ContainerExitStatus.SUCCESS: + return false; + case ContainerExitStatus.INVALID: + // Ideally, this shouldn't be considered for skipping a node. But in + // reality, it seems like there are cases where we are not setting + // exit-code correctly and so it's better to be conservative. See + // YARN-4284. + return true; + default: + return true; + } } private static final class UnmanagedAMAttemptSavedTransition @@ -1803,7 +1829,7 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, if (appAttempt.masterContainer != null && appAttempt.masterContainer.getId().equals( containerStatus.getContainerId())) { - appAttempt.sendAMContainerToNM(appAttempt, containerFinishedEvent); + appAttempt.amContainerFinished(appAttempt, containerFinishedEvent); // Remember the follow up transition and save the final attempt state. appAttempt.rememberTargetTransitionsAndStoreState(event, @@ -1848,28 +1874,31 @@ private void sendFinishedContainersToNM() { // Add am container to the list so that am container instance will be // removed from NMContext. - private void sendAMContainerToNM(RMAppAttemptImpl appAttempt, + private static void amContainerFinished(RMAppAttemptImpl appAttempt, RMAppAttemptContainerFinishedEvent containerFinishedEvent) { + NodeId nodeId = containerFinishedEvent.getNodeId(); - if (containerFinishedEvent.getContainerStatus() != null) { - if (shouldCountTowardsNodeBlacklisting(containerFinishedEvent - .getContainerStatus().getExitStatus())) { - appAttempt.addAMNodeToBlackList(containerFinishedEvent.getNodeId()); + + ContainerStatus containerStatus = + containerFinishedEvent.getContainerStatus(); + if (containerStatus != null) { + int exitStatus = containerStatus.getExitStatus(); + if (shouldCountTowardsNodeBlacklisting(exitStatus)) { + appAttempt.addAMNodeToBlackList(nodeId); } } else { LOG.warn("No ContainerStatus in containerFinishedEvent"); } if (!appAttempt.getSubmissionContext() - .getKeepContainersAcrossApplicationAttempts()) { - finishedContainersSentToAM.putIfAbsent(nodeId, - new ArrayList()); - appAttempt.finishedContainersSentToAM.get(nodeId).add( - containerFinishedEvent.getContainerStatus()); + .getKeepContainersAcrossApplicationAttempts()) { + appAttempt.finishedContainersSentToAM.putIfAbsent(nodeId, + new ArrayList()); + appAttempt.finishedContainersSentToAM.get(nodeId).add(containerStatus); appAttempt.sendFinishedContainersToNM(); } else { appAttempt.sendFinishedAMContainerToNM(nodeId, - containerFinishedEvent.getContainerStatus().getContainerId()); + containerStatus.getContainerId()); } } @@ -1884,7 +1913,7 @@ private void addAMNodeToBlackList(NodeId nodeId) { } @Override - public BlacklistManager getAMBlacklist() { + public BlacklistManager getAMBlacklistManager() { return blacklistedNodesForAM; } @@ -1943,7 +1972,7 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, containerStatus.getContainerId())) { new FinalTransition(RMAppAttemptState.FINISHED).transition( appAttempt, containerFinishedEvent); - appAttempt.sendAMContainerToNM(appAttempt, containerFinishedEvent); + appAttempt.amContainerFinished(appAttempt, containerFinishedEvent); return RMAppAttemptState.FINISHED; } // Add all finished containers so that they can be acked to NM. @@ -1968,7 +1997,8 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, // Thus, we still return FINAL_SAVING state here. if (appAttempt.masterContainer.getId().equals( containerStatus.getContainerId())) { - appAttempt.sendAMContainerToNM(appAttempt, containerFinishedEvent); + + appAttempt.amContainerFinished(appAttempt, containerFinishedEvent); if (appAttempt.targetedFinalState.equals(RMAppAttemptState.FAILED) || appAttempt.targetedFinalState.equals(RMAppAttemptState.KILLED)) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 8f03de2..ddf3aa9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -178,7 +178,7 @@ public ClusterNodeTracker getNodeTracker() { NodeFilter nodeFilter = new NodeFilter() { @Override public boolean accept(SchedulerNode node) { - return SchedulerAppUtils.isBlacklisted(app, node, LOG); + return SchedulerAppUtils.isPlaceBlacklisted(app, node, LOG); } }; return nodeTracker.getNodes(nodeFilter); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 463bebd..1a3b6b0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -73,9 +73,13 @@ private ActiveUsersManager activeUsersManager; private boolean pending = true; // whether accepted/allocated by scheduler private ResourceUsage appResourceUsage; + private AtomicBoolean userBlacklistChanged = new AtomicBoolean(false); - private final Set amBlacklist = new HashSet<>(); - private Set userBlacklist = new HashSet<>(); + // Set of places (nodes / racks) blacklisted by the system. Today, this only + // has places blacklisted for AM containers. + private final Set placesBlacklistedBySystem = new HashSet<>(); + private Set placesBlacklistedByApp = new HashSet<>(); + private Set requestedPartitions = new HashSet<>(); final Set priorities = new TreeSet<>(COMPARATOR); @@ -447,15 +451,17 @@ private boolean hasRequestLabelChanged(ResourceRequest requestOne, } /** - * The ApplicationMaster is updating the userBlacklist used for containers - * other than AMs. - * - * @param blacklistAdditions resources to be added to the userBlacklist - * @param blacklistRemovals resources to be removed from the userBlacklist + * The ApplicationMaster is updating the placesBlacklistedByApp used for + * containers other than AMs. + * + * @param blacklistAdditions + * resources to be added to the userBlacklist + * @param blacklistRemovals + * resources to be removed from the userBlacklist */ - public void updateBlacklist( + public void updatePlacesBlacklistedByApp( List blacklistAdditions, List blacklistRemovals) { - if (updateUserOrAMBlacklist(userBlacklist, blacklistAdditions, + if (updateBlacklistedPlaces(placesBlacklistedByApp, blacklistAdditions, blacklistRemovals)) { userBlacklistChanged.set(true); } @@ -466,13 +472,13 @@ public void updateBlacklist( * @param blacklistAdditions resources to be added to the amBlacklist * @param blacklistRemovals resources to be added to the amBlacklist */ - public void updateAMBlacklist( + public void updatePlacesBlacklistedBySystem( List blacklistAdditions, List blacklistRemovals) { - updateUserOrAMBlacklist(amBlacklist, blacklistAdditions, + updateBlacklistedPlaces(placesBlacklistedBySystem, blacklistAdditions, blacklistRemovals); } - boolean updateUserOrAMBlacklist(Set blacklist, + private static boolean updateBlacklistedPlaces(Set blacklist, List blacklistAdditions, List blacklistRemovals) { boolean changed = false; synchronized (blacklist) { @@ -522,20 +528,24 @@ public synchronized Resource getResource(Priority priority) { } /** - * Returns if the node is either blacklisted by the user or the system - * @param resourceName the resourcename - * @param useAMBlacklist true if it should check amBlacklist + * Returns if the place (node/rack today) is either blacklisted by the + * application (user) or the system + * + * @param resourceName + * the resourcename + * @param blacklistedBySystem + * true if it should check amBlacklist * @return true if its blacklisted */ - public boolean isBlacklisted(String resourceName, - boolean useAMBlacklist) { - if (useAMBlacklist){ - synchronized (amBlacklist) { - return amBlacklist.contains(resourceName); + public boolean isPlaceBlacklisted(String resourceName, + boolean blacklistedBySystem) { + if (blacklistedBySystem){ + synchronized (placesBlacklistedBySystem) { + return placesBlacklistedBySystem.contains(resourceName); } } else { - synchronized (userBlacklist) { - return userBlacklist.contains(resourceName); + synchronized (placesBlacklistedByApp) { + return placesBlacklistedByApp.contains(resourceName); } } } @@ -773,12 +783,12 @@ public synchronized void setQueue(Queue queue) { } public Set getBlackList() { - return this.userBlacklist; + return this.placesBlacklistedByApp; } public Set getBlackListCopy() { - synchronized (userBlacklist) { - return new HashSet<>(this.userBlacklist); + synchronized (placesBlacklistedByApp) { + return new HashSet<>(this.placesBlacklistedByApp); } } @@ -786,7 +796,7 @@ public synchronized void transferStateFromPreviousAppSchedulingInfo( AppSchedulingInfo appInfo) { // This should not require locking the userBlacklist since it will not be // used by this instance until after setCurrentAppAttempt. - this.userBlacklist = appInfo.getBlackList(); + this.placesBlacklistedByApp = appInfo.getBlackList(); } public synchronized void recoverContainer(RMContainer rmContainer) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java index 36a1244..162e4d1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java @@ -22,9 +22,9 @@ public class SchedulerAppUtils { - public static boolean isBlacklisted(SchedulerApplicationAttempt application, - SchedulerNode node, Log LOG) { - if (application.isBlacklisted(node.getNodeName())) { + public static boolean isPlaceBlacklisted( + SchedulerApplicationAttempt application, SchedulerNode node, Log LOG) { + if (application.isPlaceBlacklisted(node.getNodeName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'host' " + node.getNodeName() + " for " + application.getApplicationId() + @@ -33,7 +33,7 @@ public static boolean isBlacklisted(SchedulerApplicationAttempt application, return true; } - if (application.isBlacklisted(node.getRackName())) { + if (application.isPlaceBlacklisted(node.getRackName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'rack' " + node.getRackName() + " for " + application.getApplicationId() + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 7308e22..6c8fd88 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -592,27 +592,25 @@ public boolean isWaitingForAMContainer() { return (!unmanagedAM && appAttempt.getMasterContainer() == null); } - // Blacklist used for user containers - public synchronized void updateBlacklist( - List blacklistAdditions, List blacklistRemovals) { + public synchronized void updateBlacklist(List blacklistAdditions, + List blacklistRemovals) { if (!isStopped) { - this.appSchedulingInfo.updateBlacklist( - blacklistAdditions, blacklistRemovals); - } - } - - // Blacklist used for AM containers - public synchronized void updateAMBlacklist( - List blacklistAdditions, List blacklistRemovals) { - if (!isStopped) { - this.appSchedulingInfo.updateAMBlacklist( + if (isWaitingForAMContainer()) { + // The request is for the AM-container, so update the corresponding + // blacklists + this.appSchedulingInfo.updatePlacesBlacklistedBySystem( blacklistAdditions, blacklistRemovals); + } else { + this.appSchedulingInfo.updatePlacesBlacklistedByApp(blacklistAdditions, + blacklistRemovals); + } } } - public boolean isBlacklisted(String resourceName) { - boolean useAMBlacklist = isWaitingForAMContainer(); - return this.appSchedulingInfo.isBlacklisted(resourceName, useAMBlacklist); + public boolean isPlaceBlacklisted(String resourceName) { + boolean forAMContainer = isWaitingForAMContainer(); + return this.appSchedulingInfo.isPlaceBlacklisted(resourceName, + forAMContainer); } public synchronized int addMissedNonPartitionedRequestSchedulingOpportunity( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 920e983..2c6d742 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -997,14 +997,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, } } - if (application.isWaitingForAMContainer()) { - // Allocate is for AM and update AM blacklist for this - application.updateAMBlacklist( - blacklistAdditions, blacklistRemovals); - } else { - application.updateBlacklist(blacklistAdditions, blacklistRemovals); - } - + application.updateBlacklist(blacklistAdditions, blacklistRemovals); allocation = application.getAllocation(getResourceCalculator(), getClusterResource(), getMinimumResourceCapability()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java index a5ca2d8..420c66d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java @@ -81,7 +81,7 @@ private boolean checkHeadroom(Resource clusterResource, private ContainerAllocation preCheckForNewContainer(Resource clusterResource, FiCaSchedulerNode node, SchedulingMode schedulingMode, ResourceLimits resourceLimits, Priority priority) { - if (SchedulerAppUtils.isBlacklisted(application, node, LOG)) { + if (SchedulerAppUtils.isPlaceBlacklisted(application, node, LOG)) { application.updateAppSkipNodeDiagnostics( CSAMContainerLaunchDiagnosticsConstants.SKIP_AM_ALLOCATION_IN_BLACK_LISTED_NODE); return ContainerAllocation.APP_SKIPPED; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index ca5a146..51849f8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -331,7 +331,7 @@ public Resource assignContainer(FSSchedulerNode node) { readLock.unlock(); } for (FSAppAttempt sched : pendingForResourceApps) { - if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) { + if (SchedulerAppUtils.isPlaceBlacklisted(sched, node, LOG)) { continue; } assigned = sched.assignContainer(node); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index c59ba12..2301f70 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -991,13 +991,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, preemptionContainerIds.add(container.getContainerId()); } - if (application.isWaitingForAMContainer()) { - // Allocate is for AM and update AM blacklist for this - application.updateAMBlacklist( - blacklistAdditions, blacklistRemovals); - } else { - application.updateBlacklist(blacklistAdditions, blacklistRemovals); - } + application.updateBlacklist(blacklistAdditions, blacklistRemovals); List newlyAllocatedContainers = application.pullNewlyAllocatedContainers(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 3e6225f..9753bf3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -375,13 +375,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, " #ask=" + ask.size()); } - if (application.isWaitingForAMContainer()) { - // Allocate is for AM and update AM blacklist for this - application.updateAMBlacklist( - blacklistAdditions, blacklistRemovals); - } else { - application.updateBlacklist(blacklistAdditions, blacklistRemovals); - } + application.updateBlacklist(blacklistAdditions, blacklistRemovals); Resource headroom = application.getHeadroom(); application.setApplicationHeadroomForMetrics(headroom); @@ -516,7 +510,7 @@ private void assignContainers(FiCaSchedulerNode node) { application.showRequests(); synchronized (application) { // Check if this resource is on the blacklist - if (SchedulerAppUtils.isBlacklisted(application, node, LOG)) { + if (SchedulerAppUtils.isPlaceBlacklisted(application, node, LOG)) { continue; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 6fef367..387ef45 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -220,8 +220,9 @@ protected void generateOverview(ApplicationAttemptReport appAttemptReport, String appBlacklistedNodes = getNodeString(rmAppAttempt.getBlacklistedNodes()); // nodes which are blacklisted by the RM for AM launches - String rmBlackListedNodes = getNodeString( - rmAppAttempt.getAMBlacklist().getBlacklistUpdates().getAdditions()); + String rmBlackListedNodes = + getNodeString(rmAppAttempt.getAMBlacklistManager() + .getBlacklistUpdates().getBlacklistAdditions()); info("Application Attempt Overview") ._( @@ -257,7 +258,7 @@ protected void generateOverview(ApplicationAttemptReport appAttemptReport, appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt .getDiagnosticsInfo()) ._("Application Blacklisted Nodes:", appBlacklistedNodes) - ._("RM Blacklisted Nodes(for AM launches)", rmBlackListedNodes); + ._("Nodes skipped for AM launches:", rmBlackListedNodes); } private String getNodeString(Collection nodes) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java index 2d822c1..cfa21dd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java @@ -113,10 +113,10 @@ protected void generateApplicationTable(Block html, Hamlet.TBODY> tbody = html.table("#attempts").thead().tr().th(".id", "Attempt ID") .th(".started", "Started").th(".node", "Node").th(".logs", "Logs") - .th(".appBlacklistednodes", "Nodes black listed by the application", - "App Blacklisted Nodes") - .th(".rmBlacklistednodes", "Nodes black listed by the RM for the" - + " app", "RM Blacklisted Nodes")._()._().tbody(); + .th(".appBlacklistednodes", "Nodes blacklisted by the application", + "Nodes blacklisted by the app") + .th(".rmBlacklistednodes", "Nodes blacklisted by the RM for the" + + " app", "Nodes blacklisted by the system")._()._().tbody(); RMApp rmApp = this.rm.getRMContext().getRMApps().get(this.appID); if (rmApp == null) { @@ -136,8 +136,9 @@ protected void generateApplicationTable(Block html, // nodes which are blacklisted by the application String appBlacklistedNodesCount = String.valueOf(nodes.size()); // nodes which are blacklisted by the RM for AM launches - String rmBlacklistedNodesCount = String.valueOf(rmAppAttempt - .getAMBlacklist().getBlacklistUpdates().getAdditions().size()); + String rmBlacklistedNodesCount = + String.valueOf(rmAppAttempt.getAMBlacklistManager() + .getBlacklistUpdates().getBlacklistAdditions().size()); String nodeLink = attemptInfo.getNodeHttpAddress(); if (nodeLink != null) { nodeLink = WebAppUtils.getHttpSchemePrefix(conf) + nodeLink; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 50c450b..cacb2bf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -83,12 +83,15 @@ import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; -import org.apache.hadoop.yarn.api.records.AMBlackListingRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -128,7 +131,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AMBlackListingRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; @@ -162,14 +164,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationListInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionResponseInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionResponseInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateResponseInfo; -import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; @@ -1497,10 +1495,6 @@ protected ApplicationSubmissionContext createAppSubmissionContext( reservationIdStr); appContext.setReservationID(reservationId); } - if (newApp.getAMBlackListingRequestInfo() != null) { - appContext.setAMBlackListRequest(createAMBlackListingRequest( - newApp.getAMBlackListingRequestInfo())); - } return appContext; } @@ -1649,13 +1643,6 @@ private LogAggregationContext createLogAggregationContext( logAggregationContextInfo.getLogAggregationPolicyParameters()); } - private AMBlackListingRequest createAMBlackListingRequest( - AMBlackListingRequestInfo amBlackListingRequestInfo) { - return AMBlackListingRequest.newInstance( - amBlackListingRequestInfo.getAMBlackListingEnabled(), - amBlackListingRequestInfo.getBlackListingDisableFailureThreshold()); - } - @POST @Path("/delegation-token") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java deleted file mode 100644 index 0084b0b..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - -/** - * Simple class to allow users to send information required to create a - * AMBlackListingRequest which can then be used as part of the - * ApplicationSubmissionContext - * - */ -@XmlRootElement(name = "am-black-listing-requests") -@XmlAccessorType(XmlAccessType.FIELD) -public class AMBlackListingRequestInfo { - - @XmlElement(name = "am-black-listing-enabled") - boolean isAMBlackListingEnabled; - - @XmlElement(name = "disable-failure-threshold") - float disableFailureThreshold; - - public AMBlackListingRequestInfo() { - } - - public boolean getAMBlackListingEnabled() { - return isAMBlackListingEnabled; - } - - public void setAMBlackListingEnabled(boolean isAMBlackListingEnabled) { - this.isAMBlackListingEnabled = isAMBlackListingEnabled; - } - - public float getBlackListingDisableFailureThreshold() { - return disableFailureThreshold; - } - - public void setBlackListingDisableFailureThreshold( - float disableFailureThreshold) { - this.disableFailureThreshold = disableFailureThreshold; - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index 60b728e..f0a3afa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -42,7 +42,7 @@ protected String nodeId; protected String logsLink; protected String blacklistedNodes; - protected String rmBlacklistedNodesForAMLaunches; + protected String nodesSkippedForAMLaunches; protected String appAttemptId; public AppAttemptInfo() { @@ -69,9 +69,9 @@ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user, + masterContainer.getNodeHttpAddress(), ConverterUtils.toString(masterContainer.getId()), user); - rmBlacklistedNodesForAMLaunches = StringUtils.join( - attempt.getAMBlacklist().getBlacklistUpdates().getAdditions(), - ", "); + nodesSkippedForAMLaunches = + StringUtils.join(attempt.getAMBlacklistManager() + .getBlacklistUpdates().getBlacklistAdditions(), ", "); if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) { AbstractYarnScheduler ayScheduler = (AbstractYarnScheduler) rm.getResourceScheduler(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java index 4cbe7a8..3d95ca1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java @@ -87,9 +87,6 @@ @XmlElement(name = "reservation-id") String reservationId; - @XmlElement(name = "am-black-listing-requests") - AMBlackListingRequestInfo amBlackListingRequestInfo; - public ApplicationSubmissionContextInfo() { applicationId = ""; applicationName = ""; @@ -106,7 +103,6 @@ public ApplicationSubmissionContextInfo() { logAggregationContextInfo = null; attemptFailuresValidityInterval = -1; reservationId = ""; - amBlackListingRequestInfo = null; } public String getApplicationId() { @@ -173,10 +169,6 @@ public long getAttemptFailuresValidityInterval() { return attemptFailuresValidityInterval; } - public AMBlackListingRequestInfo getAMBlackListingRequestInfo() { - return amBlackListingRequestInfo; - } - public String getReservationId() { return reservationId; } @@ -252,9 +244,4 @@ public void setAttemptFailuresValidityInterval( public void setReservationId(String reservationId) { this.reservationId = reservationId; } - - public void setAMBlackListingRequestInfo( - AMBlackListingRequestInfo amBlackListingRequestInfo) { - this.amBlackListingRequestInfo = amBlackListingRequestInfo; - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 66780a0..ae14a2d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -277,7 +277,9 @@ public void testNMTokensRebindOnAMRestart() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3); // To prevent test from blacklisting nm1 for AM, we sit threshold to half // of 2 nodes which is 1 - conf.setFloat(YarnConfiguration.AM_BLACKLISTING_DISABLE_THRESHOLD, 0.5f); + conf.setFloat( + YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD, + 0.5f); MockRM rm1 = new MockRM(conf); rm1.start(); @@ -384,7 +386,7 @@ public void testAMBlacklistPreventsRestartOnSameNode() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); - conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true); + conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, true); testAMBlacklistPreventRestartOnSameNode(false, conf); } @@ -392,7 +394,7 @@ public void testAMBlacklistPreventsRestartOnSameNode() throws Exception { public void testAMBlacklistPreventsRestartOnSameNodeForMinicluster() throws Exception { YarnConfiguration conf = new YarnConfiguration(); - conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true); + conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, true); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true); testAMBlacklistPreventRestartOnSameNode(false, conf); @@ -401,9 +403,11 @@ public void testAMBlacklistPreventsRestartOnSameNodeForMinicluster() @Test(timeout = 100000) public void testAMBlacklistPreemption() throws Exception { YarnConfiguration conf = new YarnConfiguration(); - conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true); + conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, true); // disable the float so it is possible to blacklist the entire cluster - conf.setFloat(YarnConfiguration.AM_BLACKLISTING_DISABLE_THRESHOLD, 1.5f); + conf.setFloat( + YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD, + 1.5f); // since the exit status is PREEMPTED, it should not lead to the node being // blacklisted testAMBlacklistPreventRestartOnSameNode(true, conf); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/TestBlacklistManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/TestBlacklistManager.java index 96b373f..3102c6f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/TestBlacklistManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/TestBlacklistManager.java @@ -19,12 +19,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.blacklist; -import org.junit.Assert; -import org.junit.Test; - import java.util.Collections; import java.util.List; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; +import org.junit.Assert; +import org.junit.Test; + public class TestBlacklistManager { @Test @@ -37,12 +38,12 @@ public void testSimpleBlacklistBelowFailureThreshold() { String anyNode2 = "bar"; manager.addNode(anyNode); manager.addNode(anyNode2); - BlacklistUpdates blacklist = manager + ResourceBlacklistRequest blacklist = manager .getBlacklistUpdates(); - List blacklistAdditions = blacklist.getAdditions(); + List blacklistAdditions = blacklist.getBlacklistAdditions(); Collections.sort(blacklistAdditions); - List blacklistRemovals = blacklist.getRemovals(); + List blacklistRemovals = blacklist.getBlacklistRemovals(); String[] expectedBlacklistAdditions = new String[]{anyNode2, anyNode}; Assert.assertArrayEquals( "Blacklist additions was not as expected", @@ -61,12 +62,12 @@ public void testSimpleBlacklistAboveFailureThreshold() { String anyNode = "foo"; String anyNode2 = "bar"; manager.addNode(anyNode); - BlacklistUpdates blacklist = manager + ResourceBlacklistRequest blacklist = manager .getBlacklistUpdates(); - List blacklistAdditions = blacklist.getAdditions(); + List blacklistAdditions = blacklist.getBlacklistAdditions(); Collections.sort(blacklistAdditions); - List blacklistRemovals = blacklist.getRemovals(); + List blacklistRemovals = blacklist.getBlacklistRemovals(); String[] expectedBlacklistAdditions = new String[]{anyNode}; Assert.assertArrayEquals( "Blacklist additions was not as expected", @@ -81,9 +82,9 @@ public void testSimpleBlacklistAboveFailureThreshold() { blacklist = manager .getBlacklistUpdates(); - blacklistAdditions = blacklist.getAdditions(); + blacklistAdditions = blacklist.getBlacklistAdditions(); Collections.sort(blacklistAdditions); - blacklistRemovals = blacklist.getRemovals(); + blacklistRemovals = blacklist.getBlacklistRemovals(); Collections.sort(blacklistRemovals); String[] expectedBlacklistRemovals = new String[] {anyNode2, anyNode}; Assert.assertTrue( @@ -101,11 +102,11 @@ public void testDisabledBlacklist() { BlacklistManager disabled = new DisabledBlacklistManager(); String anyNode = "foo"; disabled.addNode(anyNode); - BlacklistUpdates blacklist = disabled + ResourceBlacklistRequest blacklist = disabled .getBlacklistUpdates(); - List blacklistAdditions = blacklist.getAdditions(); - List blacklistRemovals = blacklist.getRemovals(); + List blacklistAdditions = blacklist.getBlacklistAdditions(); + List blacklistRemovals = blacklist.getBlacklistRemovals(); Assert.assertTrue( "Blacklist additions should be empty but was " + blacklistAdditions, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 293c0b6..bb5c25c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -42,7 +42,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.yarn.MockApps; -import org.apache.hadoop.yarn.api.records.AMBlackListingRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -1029,63 +1028,6 @@ public void testGetAppReport() { + "/")); } - @Test - public void testAMBlackListConfigFromApp() { - // Scenario 1: Application enables AM blacklisting - float disableThreshold = 0.9f; - conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, false); - ApplicationSubmissionContext submissionContext = - new ApplicationSubmissionContextPBImpl(); - submissionContext.setAMBlackListRequest(AMBlackListingRequest.newInstance( - true, disableThreshold)); - RMAppImpl application = (RMAppImpl) createNewTestApp(submissionContext); - - Assert.assertTrue(application.isAmBlacklistingEnabled()); - Assert.assertEquals(disableThreshold, - application.getAmBlacklistingDisableThreshold(), 1e-8); - - // Scenario 2: Application disables AM blacklisting - float globalThreshold = 0.9f; - conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true); - conf.setFloat(YarnConfiguration.AM_BLACKLISTING_DISABLE_THRESHOLD, - globalThreshold); - ApplicationSubmissionContext submissionContext2 = - new ApplicationSubmissionContextPBImpl(); - submissionContext2.setAMBlackListRequest(AMBlackListingRequest.newInstance( - false, disableThreshold)); - RMAppImpl application2 = (RMAppImpl) createNewTestApp(submissionContext2); - - // Am blacklisting will be disabled eventhough its enabled in RM. - Assert.assertFalse(application2.isAmBlacklistingEnabled()); - - // Scenario 3: Application updates invalid AM threshold - float invalidDisableThreshold = -0.5f; - conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true); - conf.setFloat(YarnConfiguration.AM_BLACKLISTING_DISABLE_THRESHOLD, - globalThreshold); - ApplicationSubmissionContext submissionContext3 = - new ApplicationSubmissionContextPBImpl(); - submissionContext3.setAMBlackListRequest(AMBlackListingRequest.newInstance( - true, invalidDisableThreshold)); - RMAppImpl application3 = (RMAppImpl) createNewTestApp(submissionContext3); - - Assert.assertTrue(application3.isAmBlacklistingEnabled()); - Assert.assertEquals(globalThreshold, - application3.getAmBlacklistingDisableThreshold(), 1e-8); - - // Scenario 4: Empty AMBlackListingRequest in Submission Context - conf.setBoolean(YarnConfiguration.AM_BLACKLISTING_ENABLED, true); - conf.setFloat(YarnConfiguration.AM_BLACKLISTING_DISABLE_THRESHOLD, - globalThreshold); - ApplicationSubmissionContext submissionContext4 = - new ApplicationSubmissionContextPBImpl(); - RMAppImpl application4 = (RMAppImpl) createNewTestApp(submissionContext4); - - Assert.assertTrue(application4.isAmBlacklistingEnabled()); - Assert.assertEquals(globalThreshold, - application4.getAmBlacklistingDisableThreshold(), 1e-8); - } - private void verifyApplicationFinished(RMAppState state) { ArgumentCaptor finalState = ArgumentCaptor.forClass(RMAppState.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java index 4141a53..a1c6294 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java @@ -41,32 +41,32 @@ public void testBacklistChanged() { AppSchedulingInfo appSchedulingInfo = new AppSchedulingInfo( appAttemptId, "test", queue, null, 0, new ResourceUsage()); - appSchedulingInfo.updateBlacklist(new ArrayList(), + appSchedulingInfo.updatePlacesBlacklistedByApp(new ArrayList(), new ArrayList()); Assert.assertFalse(appSchedulingInfo.getAndResetBlacklistChanged()); ArrayList blacklistAdditions = new ArrayList(); blacklistAdditions.add("node1"); blacklistAdditions.add("node2"); - appSchedulingInfo.updateBlacklist(blacklistAdditions, + appSchedulingInfo.updatePlacesBlacklistedByApp(blacklistAdditions, new ArrayList()); Assert.assertTrue(appSchedulingInfo.getAndResetBlacklistChanged()); blacklistAdditions.clear(); blacklistAdditions.add("node1"); - appSchedulingInfo.updateBlacklist(blacklistAdditions, + appSchedulingInfo.updatePlacesBlacklistedByApp(blacklistAdditions, new ArrayList()); Assert.assertFalse(appSchedulingInfo.getAndResetBlacklistChanged()); ArrayList blacklistRemovals = new ArrayList(); blacklistRemovals.add("node1"); - appSchedulingInfo.updateBlacklist(new ArrayList(), + appSchedulingInfo.updatePlacesBlacklistedByApp(new ArrayList(), blacklistRemovals); - appSchedulingInfo.updateBlacklist(new ArrayList(), + appSchedulingInfo.updatePlacesBlacklistedByApp(new ArrayList(), blacklistRemovals); Assert.assertTrue(appSchedulingInfo.getAndResetBlacklistChanged()); - appSchedulingInfo.updateBlacklist(new ArrayList(), + appSchedulingInfo.updatePlacesBlacklistedByApp(new ArrayList(), blacklistRemovals); Assert.assertFalse(appSchedulingInfo.getAndResetBlacklistChanged()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 16063b5..5ffee58 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -684,11 +684,11 @@ public void testBlackListNodes() throws Exception { cs.allocate(appAttemptId, Collections.emptyList(), Collections.emptyList(), Collections.singletonList(host), null, null, null); - Assert.assertTrue(cs.getApplicationAttempt(appAttemptId).isBlacklisted(host)); + Assert.assertTrue(cs.getApplicationAttempt(appAttemptId).isPlaceBlacklisted(host)); cs.allocate(appAttemptId, Collections.emptyList(), Collections.emptyList(), null, Collections.singletonList(host), null, null); - Assert.assertFalse(cs.getApplicationAttempt(appAttemptId).isBlacklisted(host)); + Assert.assertFalse(cs.getApplicationAttempt(appAttemptId).isPlaceBlacklisted(host)); rm.stop(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java index e733b1c..29c689c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java @@ -316,24 +316,24 @@ public void testHeadroomWithBlackListedNodes() { FSAppAttempt spyApp = spy(app); doReturn(false) .when(spyApp).isWaitingForAMContainer(); - assertTrue(spyApp.isBlacklisted(n1.getNodeName())); - assertFalse(spyApp.isBlacklisted(n2.getNodeName())); + assertTrue(spyApp.isPlaceBlacklisted(n1.getNodeName())); + assertFalse(spyApp.isPlaceBlacklisted(n2.getNodeName())); assertEquals(n2.getUnallocatedResource(), spyApp.getHeadroom()); blacklistAdditions.clear(); blacklistAdditions.add(n2.getNodeName()); blacklistRemovals.add(n1.getNodeName()); app.updateBlacklist(blacklistAdditions, blacklistRemovals); - assertFalse(spyApp.isBlacklisted(n1.getNodeName())); - assertTrue(spyApp.isBlacklisted(n2.getNodeName())); + assertFalse(spyApp.isPlaceBlacklisted(n1.getNodeName())); + assertTrue(spyApp.isPlaceBlacklisted(n2.getNodeName())); assertEquals(n1.getUnallocatedResource(), spyApp.getHeadroom()); blacklistAdditions.clear(); blacklistRemovals.clear(); blacklistRemovals.add(n2.getNodeName()); app.updateBlacklist(blacklistAdditions, blacklistRemovals); - assertFalse(spyApp.isBlacklisted(n1.getNodeName())); - assertFalse(spyApp.isBlacklisted(n2.getNodeName())); + assertFalse(spyApp.isPlaceBlacklisted(n1.getNodeName())); + assertFalse(spyApp.isPlaceBlacklisted(n2.getNodeName())); assertEquals(clusterResource, spyApp.getHeadroom()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 5690fa4..c5fbc31 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3899,11 +3899,11 @@ public void testBlacklistNodes() throws Exception { scheduler.allocate(appAttemptId, Collections.emptyList(), Collections.emptyList(), Collections.singletonList(host), null, null, null); - assertTrue(app.isBlacklisted(host)); + assertTrue(app.isPlaceBlacklisted(host)); scheduler.allocate(appAttemptId, Collections.emptyList(), Collections.emptyList(), null, Collections.singletonList(host), null, null); - assertFalse(scheduler.getSchedulerApp(appAttemptId).isBlacklisted(host)); + assertFalse(scheduler.getSchedulerApp(appAttemptId).isPlaceBlacklisted(host)); List update = Arrays.asList( createResourceRequest(GB, node.getHostName(), 1, 0, true)); @@ -3912,7 +3912,7 @@ public void testBlacklistNodes() throws Exception { scheduler.allocate(appAttemptId, update, Collections.emptyList(), Collections.singletonList(host), null, null, null); - assertTrue(app.isBlacklisted(host)); + assertTrue(app.isPlaceBlacklisted(host)); scheduler.update(); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 0, app @@ -3922,7 +3922,7 @@ public void testBlacklistNodes() throws Exception { scheduler.allocate(appAttemptId, update, Collections.emptyList(), null, Collections.singletonList(host), null, null); - assertFalse(app.isBlacklisted(host)); + assertFalse(app.isPlaceBlacklisted(host)); createSchedulingRequest(GB, "root.default", "user", 1); scheduler.update(); scheduler.handle(updateEvent); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 38b32e9..c7ef8fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -22,7 +22,14 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; @@ -43,15 +50,12 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import com.sun.jersey.api.client.config.DefaultClientConfig; - import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; -import org.apache.hadoop.yarn.api.records.AMBlackListingRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -72,7 +76,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LogAggregationContextInfo; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -101,6 +111,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.LoggingFilter; import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONMarshaller; @@ -827,17 +838,6 @@ public void testAppSubmit(String acceptMedia, String contentMedia) System.currentTimeMillis(), 1).toString(); appInfo.setReservationId(reservationId); - // Set AMBlackListingRequestInfo - boolean isAMBlackListingEnabled = true; - float disableFailureThreshold = 0.01f; - AMBlackListingRequestInfo amBlackListingRequestInfo - = new AMBlackListingRequestInfo(); - amBlackListingRequestInfo.setAMBlackListingEnabled( - isAMBlackListingEnabled); - amBlackListingRequestInfo.setBlackListingDisableFailureThreshold( - disableFailureThreshold); - appInfo.setAMBlackListingRequestInfo(amBlackListingRequestInfo); - ClientResponse response = this.constructWebResource(urlPath).accept(acceptMedia) .entity(appInfo, contentMedia).post(ClientResponse.class); @@ -912,13 +912,6 @@ public void testAppSubmit(String acceptMedia, String contentMedia) // Check ReservationId assertEquals(reservationId, app.getReservationId().toString()); - // Check AMBlackListingRequestInfo - AMBlackListingRequest amBlackListingRequest = asc.getAMBlackListRequest(); - assertEquals(isAMBlackListingEnabled, - amBlackListingRequest.isAMBlackListingEnabled()); - assertTrue(disableFailureThreshold == amBlackListingRequest - .getBlackListingDisableFailureThreshold()); - response = this.constructWebResource("apps", appId).accept(acceptMedia) .get(ClientResponse.class);