diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java index e6c8c66..bca88fd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; /** @@ -60,6 +61,8 @@ * @return registration respose * @throws YarnRemoteException * @throws IOException + * @see RegisterApplicationMasterRequest + * @see RegisterApplicationMasterResponse */ public RegisterApplicationMasterResponse registerApplicationMaster( RegisterApplicationMasterRequest request) @@ -80,6 +83,8 @@ public RegisterApplicationMasterResponse registerApplicationMaster( * @return completion response * @throws YarnRemoteException * @throws IOException + * @see FinishApplicationMasterRequest + * @see FinishApplicationMasterResponse */ public FinishApplicationMasterResponse finishApplicationMaster( FinishApplicationMasterRequest request) @@ -91,12 +96,16 @@ public FinishApplicationMasterResponse finishApplicationMaster( * *

The ApplicationMaster uses this interface to provide a list * of {@link ResourceRequest} and returns unused {@link Container} allocated - * to it via {@link AllocateRequest}.

+ * to it via {@link AllocateRequest}. Optionally, the + * ApplicationMaster can also blacklist resources + * which it doesn't want to use.

* *

This also doubles up as a heartbeat to let the * ResourceManager know that the ApplicationMaster * is alive. Thus, applications should periodically make this call to be kept - * alive. The frequency depends on ??

+ * alive. The frequency depends on + * {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to + * {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.

* *

The ResourceManager responds with list of allocated * {@link Container}, status of completed containers and headroom information @@ -110,6 +119,8 @@ public FinishApplicationMasterResponse finishApplicationMaster( * @return allocation response * @throws YarnRemoteException * @throws IOException + * @see AllocateRequest + * @see AllocateResponse */ public AllocateResponse allocate(AllocateRequest request) throws YarnRemoteException, IOException; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index 766dee0..a9d6eec 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.AMRMProtocol; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.BlacklistRequest; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -113,6 +114,7 @@ * Get the list of ResourceRequest to update the * ResourceManager about the application's resource requirements. * @return the list of ResourceRequest + * @see ResourceRequest */ @Public @Stable @@ -124,6 +126,7 @@ * @param resourceRequests list of ResourceRequest to update the * ResourceManager about the application's * resource requirements + * @see ResourceRequest */ @Public @Stable @@ -143,10 +146,36 @@ * Set the list of ContainerId of containers being * released by the ApplicationMaster * @param releaseContainers list of ContainerId of - * containers being released by the < - * code>ApplicationMaster + * containers being released by the + * ApplicationMaster */ @Public @Stable void setReleaseList(List releaseContainers); + + /** + * Get the BlacklistRequest being sent by the + * ApplicationMaster. + * @return the BlacklistRequest being sent by the + * ApplicationMaster + * @see BlacklistRequest + */ + @Public + @Stable + BlacklistRequest getBlacklistRequest(); + + /** + * Set the BlacklistRequest to inform the + * ResourceManager about the blacklist additions and removals + * per the ApplicationMaster. + * + * @param blacklistRequest the BlacklistRequest to inform the + * ResourceManager about the + * blacklist additions and removals + * per the ApplicationMaster + * @see BlacklistRequest + */ + @Public + @Stable + void setBlacklistRequest(BlacklistRequest blacklistRequest); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java index 57cb77e..fbcc8bf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java @@ -25,13 +25,16 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.BlacklistRequest; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.BlacklistRequestPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.BlacklistRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto; @@ -47,6 +50,7 @@ private ApplicationAttemptId applicationAttemptID = null; private List ask = null; private List release = null; + private BlacklistRequest blacklistRequest = null; public AllocateRequestPBImpl() { @@ -75,6 +79,9 @@ private void mergeLocalToBuilder() { if (this.release != null) { addReleasesToProto(); } + if (this.blacklistRequest != null) { + builder.setBlacklistRequest(convertToProtoFormat(this.blacklistRequest)); + } } private void mergeLocalToProto() { @@ -142,6 +149,7 @@ public void setProgress(float progress) { initAsks(); return this.ask; } + @Override public void setAskList(final List resourceRequests) { if(resourceRequests == null) { @@ -152,6 +160,28 @@ public void setAskList(final List resourceRequests) { this.ask.addAll(resourceRequests); } + @Override + public BlacklistRequest getBlacklistRequest() { + AllocateRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.blacklistRequest != null) { + return this.blacklistRequest; + } + if (!p.hasBlacklistRequest()) { + return null; + } + this.blacklistRequest = convertFromProtoFormat(p.getBlacklistRequest()); + return this.blacklistRequest; + } + + @Override + public void setBlacklistRequest(BlacklistRequest blacklistRequest) { + maybeInitBuilder(); + if (blacklistRequest == null) { + builder.clearBlacklistRequest(); + } + this.blacklistRequest = blacklistRequest; + } + private void initAsks() { if (this.ask != null) { return; @@ -283,4 +313,14 @@ private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { private ContainerIdProto convertToProtoFormat(ContainerId t) { return ((ContainerIdPBImpl)t).getProto(); } + + private BlacklistRequestPBImpl convertFromProtoFormat(BlacklistRequestProto p) { + return new BlacklistRequestPBImpl(p); + } + + private BlacklistRequestProto convertToProtoFormat(BlacklistRequest t) { + return ((BlacklistRequestPBImpl)t).getProto(); + } + + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BlacklistRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BlacklistRequest.java new file mode 100644 index 0000000..897ab62 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BlacklistRequest.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.AMRMProtocol; +import org.apache.hadoop.yarn.util.Records; + +/** + * {@link BlacklistRequest} encapsulates the list of resource-names which + * should be added or removed from the blacklist of resources for the + * application. + * + * @see ResourceRequest + * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) + */ +@Public +@Stable +public abstract class BlacklistRequest { + + public static BlacklistRequest newInstance( + List additions, List removals) { + BlacklistRequest blacklistRequest = + Records.newRecord(BlacklistRequest.class); + blacklistRequest.setBlacklistAdditions(additions); + blacklistRequest.setBlacklistRemovals(removals); + return blacklistRequest; + } + + /** + * Get the list of resources which should be added to the + * application blacklist. + * + * @return list of resources which should be added to the + * application blacklist + */ + public abstract List getBlacklistAdditions(); + + /** + * Set list of resources which should be added to the application blacklist. + * + * @param resourceNames list of resources which should be added to the + * application blacklist + */ + public abstract void setBlacklistAdditions(List resourceNames); + + /** + * Get the list of resources which should be removed from the + * application blacklist. + * + * @return list of resources which should be removed from the + * application blacklist + */ + public abstract List getBlacklistRemovals(); + + /** + * Set list of resources which should be removed from the + * application blacklist. + * + * @param resourceNames list of resources which should be removed from the + * application blacklist + */ + public abstract void setBlacklistRemovals(List resourceNames); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index 32fdce7..728bebc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -38,9 +38,15 @@ * *

  • {@link Resource} required for each request.
  • *
  • - * Number of containers of such specifications which are required + * Number of containers, of above specifications, which are required * by the application. *
  • + *
  • + * A boolean relaxLocality flag, defaulting to true, + * which tells the ResourceManager if the application wants + * locality to be loose (i.e. allows fall-through to rack or any) + * or strict (i.e. specify hard constraint on resource allocation). + *
  • * *

    * diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/BlacklistRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/BlacklistRequestPBImpl.java new file mode 100644 index 0000000..736b205 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/BlacklistRequestPBImpl.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.yarn.api.records.BlacklistRequest; +import org.apache.hadoop.yarn.proto.YarnProtos.BlacklistRequestProto; +import org.apache.hadoop.yarn.proto.YarnProtos.BlacklistRequestProtoOrBuilder; + +public class BlacklistRequestPBImpl extends BlacklistRequest { + + BlacklistRequestProto proto = null; + BlacklistRequestProto.Builder builder = null; + boolean viaProto = false; + + List blacklistAdditions = null; + List blacklistRemovals = null; + + public BlacklistRequestPBImpl() { + builder = BlacklistRequestProto.newBuilder(); + } + + public BlacklistRequestPBImpl(BlacklistRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public BlacklistRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = BlacklistRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.blacklistAdditions != null) { + addBlacklistAdditionsToProto(); + } + if (this.blacklistRemovals != null) { + addBlacklistRemovalsToProto(); + } + } + + private void addBlacklistAdditionsToProto() { + maybeInitBuilder(); + builder.clearBlacklistAdditions(); + if (this.blacklistAdditions == null) { + return; + } + builder.addAllBlacklistAdditions(this.blacklistAdditions); + } + + private void addBlacklistRemovalsToProto() { + maybeInitBuilder(); + builder.clearBlacklistAdditions(); + if (this.blacklistRemovals == null) { + return; + } + builder.addAllBlacklistRemovals(this.blacklistRemovals); + } + + private void initBlacklistAdditions() { + if (this.blacklistAdditions != null) { + return; + } + BlacklistRequestProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getBlacklistAdditionsList(); + this.blacklistAdditions = new ArrayList(); + this.blacklistAdditions.addAll(list); + } + + private void initBlacklistRemovals() { + if (this.blacklistRemovals != null) { + return; + } + BlacklistRequestProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getBlacklistRemovalsList(); + this.blacklistRemovals = new ArrayList(); + this.blacklistRemovals.addAll(list); + } + + @Override + public List getBlacklistAdditions() { + initBlacklistAdditions(); + return this.blacklistAdditions; + } + + @Override + public void setBlacklistAdditions(List resourceNames) { + if (resourceNames == null) { + if (this.blacklistAdditions != null) { + this.blacklistAdditions.clear(); + } + return; + } + initBlacklistAdditions(); + this.blacklistAdditions.clear(); + this.blacklistAdditions.addAll(resourceNames); + } + + @Override + public List getBlacklistRemovals() { + initBlacklistRemovals(); + return this.blacklistRemovals; + } + + @Override + public void setBlacklistRemovals(List resourceNames) { + if (resourceNames == null) { + if (this.blacklistRemovals != null) { + this.blacklistRemovals.clear(); + } + return; + } + initBlacklistRemovals(); + this.blacklistRemovals.clear(); + this.blacklistRemovals.addAll(resourceNames); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 3bf3b2e..0104817 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -207,6 +207,11 @@ message ResourceRequestProto { optional bool relax_locality = 5 [default = true]; } +message BlacklistRequestProto { + repeated string blacklist_additions = 1; + repeated string blacklist_removals = 2; +} + //////////////////////////////////////////////////////////////////////// ////// From client_RM_Protocol ///////////////////////////////////////// //////////////////////////////////////////////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index ed3f871..4a36967 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -54,8 +54,9 @@ message AllocateRequestProto { optional ApplicationAttemptIdProto application_attempt_id = 1; repeated ResourceRequestProto ask = 2; repeated ContainerIdProto release = 3; - optional int32 response_id = 4; - optional float progress = 5; + optional BlacklistRequestProto blacklist_request = 4; + optional int32 response_id = 5; + optional float progress = 6; } message AllocateResponseProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 3094a93..3e1ab74 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidBlacklistRequestException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidResourceRequestException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; @@ -295,9 +296,19 @@ public AllocateResponse allocate(AllocateRequest request) LOG.warn("Invalid resource ask by application " + appAttemptId, e); throw RPCUtil.getRemoteException(e); } + + try { + SchedulerUtils.validateBlacklistRequest(request.getBlacklistRequest()); + } catch (InvalidBlacklistRequestException e) { + LOG.warn("Invalid blacklist request by application " + appAttemptId, e); + throw RPCUtil.getRemoteException(e); + } + // Send new requests to appAttempt. Allocation allocation = - this.rScheduler.allocate(appAttemptId, ask, release); + this.rScheduler.allocate(appAttemptId, ask, release, + request.getBlacklistRequest().getBlacklistAdditions(), + request.getBlacklistRequest().getBlacklistRemovals()); RMApp app = this.rmContext.getRMApps().get( appAttemptId.getApplicationId()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 13bbe42..7d0220d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -803,7 +803,7 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, // AM resource has been checked when submission Allocation amContainerAllocation = appAttempt.scheduler.allocate( appAttempt.applicationAttemptId, - Collections.singletonList(request), EMPTY_CONTAINER_RELEASE_LIST); + Collections.singletonList(request), EMPTY_CONTAINER_RELEASE_LIST, null, null); if (amContainerAllocation != null && amContainerAllocation.getContainers() != null) { assert (amContainerAllocation.getContainers().size() == 0); @@ -827,7 +827,7 @@ public void transition(RMAppAttemptImpl appAttempt, // Acquire the AM container from the scheduler. Allocation amContainerAllocation = appAttempt.scheduler.allocate( appAttempt.applicationAttemptId, EMPTY_CONTAINER_REQUEST_LIST, - EMPTY_CONTAINER_RELEASE_LIST); + EMPTY_CONTAINER_RELEASE_LIST, null, null); // Set the masterContainer appAttempt.setMasterContainer(amContainerAllocation.getContainers().get( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 6962a2c..416e1e6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -59,7 +60,8 @@ new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator()); final Map> requests = new HashMap>(); - + final Set blacklist = new HashSet(); + //private final ApplicationStore store; private final ActiveUsersManager activeUsersManager; @@ -114,12 +116,15 @@ public int getNewContainerId() { * application, by asking for more resources and releasing resources acquired * by the application. * - * @param requests - * resources to be acquired + * @param requests resources to be acquired + * @param blacklistAdditions resources to be added to the blacklist + * @param blacklistRemovals resources to be removed from the blacklist */ synchronized public void updateResourceRequests( - List requests) { + List requests, + List blacklistAdditions, List blacklistRemovals) { QueueMetrics metrics = queue.getMetrics(); + // Update resource requests for (ResourceRequest request : requests) { Priority priority = request.getPriority(); @@ -175,6 +180,20 @@ synchronized public void updateResourceRequests( lastRequestContainers))); } } + + // + // Update blacklist + // + + // Add to blacklist + if (blacklistAdditions != null) { + blacklist.addAll(blacklistAdditions); + } + + // Remove from blacklist + if (blacklistRemovals != null) { + blacklist.removeAll(blacklistRemovals); + } } synchronized public Collection getPriorities() { @@ -197,6 +216,10 @@ public synchronized Resource getResource(Priority priority) { return request.getCapability(); } + public synchronized boolean isBlacklisted(String resourceName) { + return blacklist.contains(resourceName); + } + /** * Resources have been allocated to this application by the resource * scheduler. Track them. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidBlacklistRequestException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidBlacklistRequestException.java new file mode 100644 index 0000000..2087ebd --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidBlacklistRequestException.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.yarn.YarnException; + +/** + * The exception is thrown when the requested resource is out of the range + * of the configured lower and upper resource boundaries. + * + */ +public class InvalidBlacklistRequestException extends YarnException { + + private static final long serialVersionUID = 384957911L; + + public InvalidBlacklistRequestException(Throwable cause) { + super(cause); + } + + public InvalidBlacklistRequestException(String message) { + super(message); + } + + public InvalidBlacklistRequestException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java index 3d1e7dd..85c17bb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java @@ -19,14 +19,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.yarn.YarnException; +import org.apache.hadoop.yarn.api.records.ResourceRequest; /** - * The exception is thrown when the requested resource is out of the range - * of the configured lower and upper resource boundaries. - * + * The exception is thrown when an application tries to blacklist + * {@link ResourceRequest#ANY}. */ public class InvalidResourceRequestException extends YarnException { + private static final long serialVersionUID = 13498237L; + public InvalidResourceRequestException(Throwable cause) { super(cause); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index c0a54c7..5d6b8ee 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.BlacklistRequest; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -152,4 +153,14 @@ public static void validateResourceRequests(List ask, } } + public static void validateBlacklistRequest(BlacklistRequest blacklistRequest) + throws InvalidBlacklistRequestException { + if (blacklistRequest != null) { + List plus = blacklistRequest.getBlacklistAdditions(); + if (plus != null && plus.contains(ResourceRequest.ANY)) { + throw new InvalidBlacklistRequestException( + "Cannot add " + ResourceRequest.ANY + " to the blacklist!"); + } + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index f084649..08f667c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -95,6 +95,8 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, * @param appAttemptId * @param ask * @param release + * @param blacklistAdditions + * @param blacklistRemovals * @return the {@link Allocation} for the application */ @Public @@ -102,7 +104,9 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, Allocation allocate(ApplicationAttemptId appAttemptId, List ask, - List release); + List release, + List blacklistAdditions, + List blacklistRemovals); /** * Get node resource usage report. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index aca2a12..4ebf94b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -472,7 +472,8 @@ private synchronized void doneApplication( @Override @Lock(Lock.NoLock.class) public Allocation allocate(ApplicationAttemptId applicationAttemptId, - List ask, List release) { + List ask, List release, + List blacklistAdditions, List blacklistRemovals) { FiCaSchedulerApp application = getApplication(applicationAttemptId); if (application == null) { @@ -523,7 +524,8 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, application.showRequests(); // Update application requests - application.updateResourceRequests(ask); + application.updateResourceRequests(ask, + blacklistAdditions, blacklistRemovals); LOG.debug("allocate: post-update"); application.showRequests(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 1d0cec2..a8a1acf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -815,6 +815,11 @@ private synchronized FiCaSchedulerApp getApplication( } synchronized (application) { + // Check if this resource is on the blacklist + if (isBlacklisted(application, node)) { + continue; + } + // Schedule in priority order for (Priority priority : application.getPriorities()) { // Required resource @@ -897,6 +902,28 @@ private synchronized FiCaSchedulerApp getApplication( return NULL_ASSIGNMENT; } + + boolean isBlacklisted(FiCaSchedulerApp application, FiCaSchedulerNode node) { + if (application.isBlacklisted(node.getHostName())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping 'host' " + node.getHostName() + + " for " + application.getApplicationId() + + " since it has been blacklisted"); + } + return true; + } + + if (application.isBlacklisted(node.getRackName())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping 'rack' " + node.getRackName() + + " for " + application.getApplicationId() + + " since it has been blacklisted"); + } + return true; + } + + return false; + } private synchronized CSAssignment assignReservedContainer(FiCaSchedulerApp application, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index ba4efaa..86bed0d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -134,9 +134,11 @@ public String getUser() { } public synchronized void updateResourceRequests( - List requests) { + List requests, + List blacklistAdditions, List blacklistRemovals) { if (!isStopped) { - this.appSchedulingInfo.updateResourceRequests(requests); + this.appSchedulingInfo.updateResourceRequests(requests, + blacklistAdditions, blacklistRemovals); } } @@ -163,6 +165,10 @@ public synchronized int getTotalRequiredResources(Priority priority) { public Resource getResource(Priority priority) { return this.appSchedulingInfo.getResource(priority); } + + public boolean isBlacklisted(String resourceName) { + return this.appSchedulingInfo.isBlacklisted(resourceName); + } /** * Is this application pending? diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index 9db15a6..4aaf74b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -138,7 +138,7 @@ public String getUser() { public synchronized void updateResourceRequests( List requests) { - this.appSchedulingInfo.updateResourceRequests(requests); + this.appSchedulingInfo.updateResourceRequests(requests, null, null); } public Map getResourceRequests(Priority priority) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index f8cff0e..b996952 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -718,7 +718,7 @@ private synchronized void removeNode(RMNode rmNode) { @Override public Allocation allocate(ApplicationAttemptId appAttemptId, - List ask, List release) { + List ask, List release, List blacklistAdditions, List blacklistRemovals) { // Make sure this application exists FSSchedulerApp application = applications.get(appAttemptId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 2f4e70d..4725406 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -222,7 +222,7 @@ public Resource getMaximumResourceCapability() { @Override public Allocation allocate( ApplicationAttemptId applicationAttemptId, List ask, - List release) { + List release, List blacklistAdditions, List blacklistRemovals) { FiCaSchedulerApp application = getApplication(applicationAttemptId); if (application == null) { LOG.error("Calling allocate on removed " + @@ -268,7 +268,7 @@ public Allocation allocate( application.showRequests(); // Update application requests - application.updateResourceRequests(ask); + application.updateResourceRequests(ask, null, null); LOG.debug("allocate: post-update" + " applicationId=" + applicationAttemptId + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index 8e49144..764f8ae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -265,7 +265,7 @@ private synchronized void addResourceRequest( // Get resources from the ResourceManager resourceManager.getResourceScheduler().allocate(applicationAttemptId, - new ArrayList(ask), new ArrayList()); + new ArrayList(ask), new ArrayList(), null, null); System.out.println("-=======" + applicationAttemptId); System.out.println("----------" + resourceManager.getRMContext().getRMApps() .get(applicationId).getRMAppAttempt(applicationAttemptId)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 9106aab..667248d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -273,23 +273,23 @@ public void testHeadroom() throws Exception { List ask1 = new ArrayList(); ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1)); - fs.allocate(appAttemptId1, ask1, emptyId); + fs.allocate(appAttemptId1, ask1, emptyId, null, null); // Ask for a 2 GB container for app 2 List ask2 = new ArrayList(); ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1)); - fs.allocate(appAttemptId2, ask2, emptyId); + fs.allocate(appAttemptId2, ask2, emptyId, null, null); // Trigger container assignment fs.handle(new NodeUpdateSchedulerEvent(n1)); // Get the allocation for the applications and verify headroom - Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId); + Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null); Assert.assertEquals("Allocation headroom", 1 * GB, allocation1.getResourceLimit().getMemory()); - Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId); + Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null); Assert.assertEquals("Allocation headroom", 1 * GB, allocation2.getResourceLimit().getMemory()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 8de3397..c5b0674 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -330,7 +330,7 @@ private void testAppAttemptScheduledState() { applicationAttempt.getAppAttemptState()); verify(scheduler, times(expectedAllocateCount)). allocate(any(ApplicationAttemptId.class), - any(List.class), any(List.class)); + any(List.class), any(List.class), null, null); assertEquals(0,applicationAttempt.getJustFinishedContainers().size()); assertNull(applicationAttempt.getMasterContainer()); @@ -354,7 +354,7 @@ private void testAppAttemptAllocatedState(Container amContainer) { verify(applicationMasterLauncher).handle(any(AMLauncherEvent.class)); verify(scheduler, times(2)). allocate( - any(ApplicationAttemptId.class), any(List.class), any(List.class)); + any(ApplicationAttemptId.class), any(List.class), any(List.class), null, null); } /** @@ -481,7 +481,7 @@ private Container allocateApplicationAttempt() { scheduler.allocate( any(ApplicationAttemptId.class), any(List.class), - any(List.class))). + any(List.class), null, null)). thenReturn(allocation); applicationAttempt.handle( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 84d9236..f8dd885 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -513,7 +513,7 @@ public void testHeadroom() throws Exception { app_0_0_requests.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, priority_1, recordFactory)); - app_0_0.updateResourceRequests(app_0_0_requests); + app_0_0.updateResourceRequests(app_0_0_requests, null, null); // Schedule to compute queue.assignContainers(clusterResource, node_0); @@ -532,7 +532,7 @@ public void testHeadroom() throws Exception { app_0_1_requests.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, priority_1, recordFactory)); - app_0_1.updateResourceRequests(app_0_1_requests); + app_0_1.updateResourceRequests(app_0_1_requests, null, null); // Schedule to compute queue.assignContainers(clusterResource, node_0); // Schedule to compute @@ -551,7 +551,7 @@ public void testHeadroom() throws Exception { app_1_0_requests.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, priority_1, recordFactory)); - app_1_0.updateResourceRequests(app_1_0_requests); + app_1_0.updateResourceRequests(app_1_0_requests, null, null); // Schedule to compute queue.assignContainers(clusterResource, node_0); // Schedule to compute diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 908e9a8..35d9d16 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -43,6 +43,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.BlacklistRequest; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -293,7 +294,7 @@ public void testSingleQueueOneUserMetrics() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Start testing... @@ -415,11 +416,11 @@ public void testSingleQueueWithOneUser() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Start testing... @@ -548,11 +549,11 @@ public void testUserLimits() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); /** * Start testing... @@ -641,11 +642,11 @@ public void testHeadroomWithMaxCap() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); /** * Start testing... @@ -680,7 +681,7 @@ public void testHeadroomWithMaxCap() throws Exception { a.setMaxCapacity(.1f); app_2.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); assertEquals(2, a.getActiveUsersManager().getNumActiveUsers()); // No more to user_0 since he is already over user-limit @@ -697,7 +698,7 @@ public void testHeadroomWithMaxCap() throws Exception { LOG.info("here"); app_1.updateResourceRequests(Collections.singletonList( // unset TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, true, - priority, recordFactory))); + priority, recordFactory)), null, null); assertEquals(1, a.getActiveUsersManager().getNumActiveUsers()); a.assignContainers(clusterResource, node_1); assertEquals(1*GB, app_2.getHeadroom().getMemory()); // hit queue max-cap @@ -758,11 +759,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true, - priority, recordFactory))); + priority, recordFactory)), null, null); /** * Start testing... @@ -792,11 +793,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception { app_2.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_3.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Now allocations should goto app_2 since // user_0 is at limit inspite of high user-limit-factor @@ -920,11 +921,11 @@ public void testReservation() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Start testing... @@ -1022,7 +1023,7 @@ public void testStolenReservedContainer() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Setup app_1 to request a 4GB container on host_0 and // another 4GB container anywhere. @@ -1034,7 +1035,7 @@ public void testStolenReservedContainer() throws Exception { true, priority, recordFactory)); appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2, true, priority, recordFactory)); - app_1.updateResourceRequests(appRequests_1); + app_1.updateResourceRequests(appRequests_1, null, null); // Start testing... @@ -1128,11 +1129,11 @@ public void testReservationExchange() throws Exception { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Start testing... @@ -1255,7 +1256,7 @@ public void testLocalityScheduling() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // Start testing... CSAssignment assignment = null; @@ -1320,7 +1321,7 @@ public void testLocalityScheduling() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); assertEquals(2, app_0.getTotalRequiredResources(priority)); String host_3 = "127.0.0.4"; // on rack_1 @@ -1411,7 +1412,7 @@ public void testApplicationPriorityScheduling() throws Exception { TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, priority_2, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // Start testing... @@ -1526,7 +1527,7 @@ public void testSchedulingConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(rack_1, 1*GB, 1, true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // Start testing... @@ -1535,7 +1536,7 @@ public void testSchedulingConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // NODE_LOCAL - node_0_1 a.assignContainers(clusterResource, node_0_0); @@ -1558,7 +1559,7 @@ public void testSchedulingConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // No allocation on node_0_1 even though it's node/rack local since // required(rack_1) == 0 @@ -1759,7 +1760,6 @@ public void testLocalityConstraints() throws Exception { // Setup some nodes and racks String host_0_0 = "127.0.0.1"; String rack_0 = "rack_0"; - FiCaSchedulerNode node_0_0 = TestUtils.getMockNode(host_0_0, rack_0, 0, 8*GB); String host_0_1 = "127.0.0.2"; FiCaSchedulerNode node_0_1 = TestUtils.getMockNode(host_0_1, rack_0, 0, 8*GB); @@ -1775,6 +1775,10 @@ public void testLocalityConstraints() throws Exception { numNodes * (8*GB), numNodes * 1); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + // Blacklist + BlacklistRequest blacklistRequest = + BlacklistRequest.newInstance(null, null); + // Setup resource-requests // resourceName: // host_0_0: < 1, 1GB, 1, true > @@ -1789,6 +1793,7 @@ public void testLocalityConstraints() throws Exception { // host_0_1: 8G // host_1_0: 8G // host_1_1: 8G + // Blacklist: Priority priority = TestUtils.createMockPriority(1); List app_0_requests_0 = new ArrayList(); app_0_requests_0.add( @@ -1803,8 +1808,10 @@ public void testLocalityConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one false, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + blacklistRequest.setBlacklistAdditions(Collections.singletonList(host_0_0)); + app_0.updateResourceRequests(app_0_requests_0, null, null); app_0_requests_0.clear(); + blacklistRequest.setBlacklistAdditions(null); // reset // // Start testing... @@ -1838,12 +1845,14 @@ public void testLocalityConstraints() throws Exception { any(Priority.class), any(ResourceRequest.class), any(Container.class)); assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0 - // Allow rack-locality for rack_1 + // Allow rack-locality for rack_1, but blacklist node_1_1 app_0_requests_0.add( TestUtils.createResourceRequest(rack_1, 1*GB, 1, true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + blacklistRequest.setBlacklistAdditions(Collections.singletonList(host_1_1)); + app_0.updateResourceRequests(app_0_requests_0, null, null); app_0_requests_0.clear(); + blacklistRequest.setBlacklistAdditions(null); // reset // resourceName: // host_0_0: < 1, 1GB, 1, true > @@ -1860,6 +1869,33 @@ public void testLocalityConstraints() throws Exception { // host_1_1: 8G // node_1_1 + // Shouldn't allocate since node_1_1 is blacklisted + a.assignContainers(clusterResource, node_1_1); + verify(app_0, never()).allocate(any(NodeType.class), eq(node_0_1), + any(Priority.class), any(ResourceRequest.class), any(Container.class)); + assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0 + + // Now, remove node_1_1 from blacklist, but add rack_1 to blacklist + blacklistRequest.setBlacklistAdditions(Collections.singletonList(rack_1)); + blacklistRequest.setBlacklistRemovals(Collections.singletonList(host_1_1)); + app_0.updateResourceRequests(app_0_requests_0, null, null); + app_0_requests_0.clear(); + blacklistRequest.setBlacklistAdditions(null); + blacklistRequest.setBlacklistRemovals(null); + + // node_1_1 + // Shouldn't allocate since rack_1 is blacklisted + a.assignContainers(clusterResource, node_1_1); + verify(app_0, never()).allocate(any(NodeType.class), eq(node_0_1), + any(Priority.class), any(ResourceRequest.class), any(Container.class)); + assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0 + + // Now clear blacklist + blacklistRequest.setBlacklistRemovals(Collections.singletonList(rack_1)); + app_0.updateResourceRequests(app_0_requests_0, null, null); + app_0_requests_0.clear(); + blacklistRequest.setBlacklistRemovals(null); + // Now, should allocate since RR(rack_1) = relax: true a.assignContainers(clusterResource, node_1_1); verify(app_0).allocate(eq(NodeType.RACK_LOCAL), eq(node_1_1), @@ -1874,7 +1910,7 @@ public void testLocalityConstraints() throws Exception { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one false, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); app_0_requests_0.clear(); // resourceName: diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index e2c753f..b4119ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -206,7 +206,7 @@ private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, priority, numContainers, true); ask.add(request); - scheduler.allocate(id, ask, new ArrayList()); + scheduler.allocate(id, ask, new ArrayList(), null, null); return id; } @@ -221,7 +221,7 @@ private void createSchedulingRequestExistingApplication(ResourceRequest request, ApplicationAttemptId attId) { List ask = new ArrayList(); ask.add(request); - scheduler.allocate(attId, ask, new ArrayList()); + scheduler.allocate(attId, ask, new ArrayList(), null, null); } // TESTS @@ -528,7 +528,7 @@ public void testQueueDemandCalculation() throws Exception { ResourceRequest request1 = createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1, true); ask1.add(request1); - scheduler.allocate(id11, ask1, new ArrayList()); + scheduler.allocate(id11, ask1, new ArrayList(), null, null); // Second ask, queue2 requests 1 large + (2 * minReqSize) List ask2 = new ArrayList(); @@ -538,14 +538,14 @@ public void testQueueDemandCalculation() throws Exception { false); ask2.add(request2); ask2.add(request3); - scheduler.allocate(id21, ask2, new ArrayList()); + scheduler.allocate(id21, ask2, new ArrayList(), null, null); // Third ask, queue2 requests 1 large List ask3 = new ArrayList(); ResourceRequest request4 = createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1, true); ask3.add(request4); - scheduler.allocate(id22, ask3, new ArrayList()); + scheduler.allocate(id22, ask3, new ArrayList(), null, null); scheduler.update(); @@ -1369,7 +1369,7 @@ public void testReservationWhileMultiplePriorities() { // Complete container scheduler.allocate(attId, new ArrayList(), - Arrays.asList(containerId)); + Arrays.asList(containerId), null, null); assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); // Schedule at opening @@ -1444,7 +1444,7 @@ public void testMultipleNodesSingleRackRequest() throws Exception { asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1, true)); asks.add(createResourceRequest(1024, ResourceRequest.ANY, 1, 2, true)); - scheduler.allocate(appId, asks, new ArrayList()); + scheduler.allocate(appId, asks, new ArrayList(), null, null); // node 1 checks in scheduler.update(); @@ -1799,7 +1799,7 @@ public void testCancelStrictLocality() { createResourceRequest(1024, node1.getHostName(), 1, 0, true), createResourceRequest(1024, "rack1", 1, 0, true), createResourceRequest(1024, ResourceRequest.ANY, 1, 1, true)); - scheduler.allocate(attId1, update, new ArrayList()); + scheduler.allocate(attId1, update, new ArrayList(), null, null); // then node2 should get the container scheduler.handle(node2UpdateEvent); @@ -1842,7 +1842,7 @@ public void testReservationsStrictLocality() { anyRequest = createResourceRequest(1024, ResourceRequest.ANY, 1, 1, false); scheduler.allocate(attId, Arrays.asList(rackRequest, anyRequest), - new ArrayList()); + new ArrayList(), null, null); scheduler.handle(nodeUpdateEvent); assertEquals(0, app.getReservedContainers().size()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index d252421..698e26f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -186,7 +186,7 @@ public void testNodeLocalAssignment() throws Exception { ask.add(nodeLocal); ask.add(rackLocal); ask.add(any); - scheduler.allocate(appAttemptId, ask, new ArrayList()); + scheduler.allocate(appAttemptId, ask, new ArrayList(), null, null); NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);