diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java
index e6c8c66..bca88fd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
/**
@@ -60,6 +61,8 @@
* @return registration respose
* @throws YarnRemoteException
* @throws IOException
+ * @see RegisterApplicationMasterRequest
+ * @see RegisterApplicationMasterResponse
*/
public RegisterApplicationMasterResponse registerApplicationMaster(
RegisterApplicationMasterRequest request)
@@ -80,6 +83,8 @@ public RegisterApplicationMasterResponse registerApplicationMaster(
* @return completion response
* @throws YarnRemoteException
* @throws IOException
+ * @see FinishApplicationMasterRequest
+ * @see FinishApplicationMasterResponse
*/
public FinishApplicationMasterResponse finishApplicationMaster(
FinishApplicationMasterRequest request)
@@ -91,12 +96,16 @@ public FinishApplicationMasterResponse finishApplicationMaster(
*
*
The ApplicationMaster uses this interface to provide a list
* of {@link ResourceRequest} and returns unused {@link Container} allocated
- * to it via {@link AllocateRequest}.
+ * to it via {@link AllocateRequest}. Optionally, the
+ * ApplicationMaster can also blacklist resources
+ * which it doesn't want to use.
*
* This also doubles up as a heartbeat to let the
* ResourceManager know that the ApplicationMaster
* is alive. Thus, applications should periodically make this call to be kept
- * alive. The frequency depends on ??
+ * alive. The frequency depends on
+ * {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to
+ * {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.
*
* The ResourceManager responds with list of allocated
* {@link Container}, status of completed containers and headroom information
@@ -110,6 +119,8 @@ public FinishApplicationMasterResponse finishApplicationMaster(
* @return allocation response
* @throws YarnRemoteException
* @throws IOException
+ * @see AllocateRequest
+ * @see AllocateResponse
*/
public AllocateResponse allocate(AllocateRequest request)
throws YarnRemoteException, IOException;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index 766dee0..a9d6eec 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.AMRMProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -113,6 +114,7 @@
* Get the list of ResourceRequest to update the
* ResourceManager about the application's resource requirements.
* @return the list of ResourceRequest
+ * @see ResourceRequest
*/
@Public
@Stable
@@ -124,6 +126,7 @@
* @param resourceRequests list of ResourceRequest to update the
* ResourceManager about the application's
* resource requirements
+ * @see ResourceRequest
*/
@Public
@Stable
@@ -143,10 +146,36 @@
* Set the list of ContainerId of containers being
* released by the ApplicationMaster
* @param releaseContainers list of ContainerId of
- * containers being released by the <
- * code>ApplicationMaster
+ * containers being released by the
+ * ApplicationMaster
*/
@Public
@Stable
void setReleaseList(List releaseContainers);
+
+ /**
+ * Get the BlacklistRequest being sent by the
+ * ApplicationMaster.
+ * @return the BlacklistRequest being sent by the
+ * ApplicationMaster
+ * @see BlacklistRequest
+ */
+ @Public
+ @Stable
+ BlacklistRequest getBlacklistRequest();
+
+ /**
+ * Set the BlacklistRequest to inform the
+ * ResourceManager about the blacklist additions and removals
+ * per the ApplicationMaster.
+ *
+ * @param blacklistRequest the BlacklistRequest to inform the
+ * ResourceManager about the
+ * blacklist additions and removals
+ * per the ApplicationMaster
+ * @see BlacklistRequest
+ */
+ @Public
+ @Stable
+ void setBlacklistRequest(BlacklistRequest blacklistRequest);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
index 57cb77e..a26fbcd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
@@ -25,13 +25,16 @@
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.BlacklistRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.BlacklistRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto;
@@ -47,6 +50,7 @@
private ApplicationAttemptId applicationAttemptID = null;
private List ask = null;
private List release = null;
+ private BlacklistRequest blacklistRequest = null;
public AllocateRequestPBImpl() {
@@ -75,6 +79,9 @@ private void mergeLocalToBuilder() {
if (this.release != null) {
addReleasesToProto();
}
+ if (this.blacklistRequest != null) {
+ builder.setBlacklistRequest(convertToProtoFormat(this.blacklistRequest));
+ }
}
private void mergeLocalToProto() {
@@ -142,6 +149,7 @@ public void setProgress(float progress) {
initAsks();
return this.ask;
}
+
@Override
public void setAskList(final List resourceRequests) {
if(resourceRequests == null) {
@@ -152,6 +160,28 @@ public void setAskList(final List resourceRequests) {
this.ask.addAll(resourceRequests);
}
+ @Override
+ public BlacklistRequest getBlacklistRequest() {
+ AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.blacklistRequest != null) {
+ return this.blacklistRequest;
+ }
+ if (!p.hasBlacklistRequest()) {
+ return null;
+ }
+ this.blacklistRequest = convertFromProtoFormat(p.getBlacklistRequest());
+ return this.blacklistRequest;
+ }
+
+ @Override
+ public void setBlacklistRequest(BlacklistRequest blacklistRequest) {
+ maybeInitBuilder();
+ if (this.blacklistRequest == null) {
+ builder.clearBlacklistRequest();
+ }
+ this.blacklistRequest = blacklistRequest;
+ }
+
private void initAsks() {
if (this.ask != null) {
return;
@@ -283,4 +313,14 @@ private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
private ContainerIdProto convertToProtoFormat(ContainerId t) {
return ((ContainerIdPBImpl)t).getProto();
}
+
+ private BlacklistRequestPBImpl convertFromProtoFormat(BlacklistRequestProto p) {
+ return new BlacklistRequestPBImpl(p);
+ }
+
+ private BlacklistRequestProto convertToProtoFormat(BlacklistRequest t) {
+ return ((BlacklistRequestPBImpl)t).getProto();
+ }
+
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BlacklistRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BlacklistRequest.java
new file mode 100644
index 0000000..897ab62
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BlacklistRequest.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@link BlacklistRequest} encapsulates the list of resource-names which
+ * should be added or removed from the blacklist of resources for the
+ * application.
+ *
+ * @see ResourceRequest
+ * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ */
+@Public
+@Stable
+public abstract class BlacklistRequest {
+
+ public static BlacklistRequest newInstance(
+ List additions, List removals) {
+ BlacklistRequest blacklistRequest =
+ Records.newRecord(BlacklistRequest.class);
+ blacklistRequest.setBlacklistAdditions(additions);
+ blacklistRequest.setBlacklistRemovals(removals);
+ return blacklistRequest;
+ }
+
+ /**
+ * Get the list of resources which should be added to the
+ * application blacklist.
+ *
+ * @return list of resources which should be added to the
+ * application blacklist
+ */
+ public abstract List getBlacklistAdditions();
+
+ /**
+ * Set list of resources which should be added to the application blacklist.
+ *
+ * @param resourceNames list of resources which should be added to the
+ * application blacklist
+ */
+ public abstract void setBlacklistAdditions(List resourceNames);
+
+ /**
+ * Get the list of resources which should be removed from the
+ * application blacklist.
+ *
+ * @return list of resources which should be removed from the
+ * application blacklist
+ */
+ public abstract List getBlacklistRemovals();
+
+ /**
+ * Set list of resources which should be removed from the
+ * application blacklist.
+ *
+ * @param resourceNames list of resources which should be removed from the
+ * application blacklist
+ */
+ public abstract void setBlacklistRemovals(List resourceNames);
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 32fdce7..728bebc 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -38,9 +38,15 @@
*
* {@link Resource} required for each request.
*
- * Number of containers of such specifications which are required
+ * Number of containers, of above specifications, which are required
* by the application.
*
+ *
+ * A boolean relaxLocality flag, defaulting to true,
+ * which tells the ResourceManager if the application wants
+ * locality to be loose (i.e. allows fall-through to rack or any)
+ * or strict (i.e. specify hard constraint on resource allocation).
+ *
*
*
*
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/BlacklistRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/BlacklistRequestPBImpl.java
new file mode 100644
index 0000000..736b205
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/BlacklistRequestPBImpl.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
+import org.apache.hadoop.yarn.proto.YarnProtos.BlacklistRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.BlacklistRequestProtoOrBuilder;
+
+public class BlacklistRequestPBImpl extends BlacklistRequest {
+
+ BlacklistRequestProto proto = null;
+ BlacklistRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ List blacklistAdditions = null;
+ List blacklistRemovals = null;
+
+ public BlacklistRequestPBImpl() {
+ builder = BlacklistRequestProto.newBuilder();
+ }
+
+ public BlacklistRequestPBImpl(BlacklistRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public BlacklistRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = BlacklistRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.blacklistAdditions != null) {
+ addBlacklistAdditionsToProto();
+ }
+ if (this.blacklistRemovals != null) {
+ addBlacklistRemovalsToProto();
+ }
+ }
+
+ private void addBlacklistAdditionsToProto() {
+ maybeInitBuilder();
+ builder.clearBlacklistAdditions();
+ if (this.blacklistAdditions == null) {
+ return;
+ }
+ builder.addAllBlacklistAdditions(this.blacklistAdditions);
+ }
+
+ private void addBlacklistRemovalsToProto() {
+ maybeInitBuilder();
+ builder.clearBlacklistAdditions();
+ if (this.blacklistRemovals == null) {
+ return;
+ }
+ builder.addAllBlacklistRemovals(this.blacklistRemovals);
+ }
+
+ private void initBlacklistAdditions() {
+ if (this.blacklistAdditions != null) {
+ return;
+ }
+ BlacklistRequestProtoOrBuilder p = viaProto ? proto : builder;
+ List list = p.getBlacklistAdditionsList();
+ this.blacklistAdditions = new ArrayList();
+ this.blacklistAdditions.addAll(list);
+ }
+
+ private void initBlacklistRemovals() {
+ if (this.blacklistRemovals != null) {
+ return;
+ }
+ BlacklistRequestProtoOrBuilder p = viaProto ? proto : builder;
+ List list = p.getBlacklistRemovalsList();
+ this.blacklistRemovals = new ArrayList();
+ this.blacklistRemovals.addAll(list);
+ }
+
+ @Override
+ public List getBlacklistAdditions() {
+ initBlacklistAdditions();
+ return this.blacklistAdditions;
+ }
+
+ @Override
+ public void setBlacklistAdditions(List resourceNames) {
+ if (resourceNames == null) {
+ if (this.blacklistAdditions != null) {
+ this.blacklistAdditions.clear();
+ }
+ return;
+ }
+ initBlacklistAdditions();
+ this.blacklistAdditions.clear();
+ this.blacklistAdditions.addAll(resourceNames);
+ }
+
+ @Override
+ public List getBlacklistRemovals() {
+ initBlacklistRemovals();
+ return this.blacklistRemovals;
+ }
+
+ @Override
+ public void setBlacklistRemovals(List resourceNames) {
+ if (resourceNames == null) {
+ if (this.blacklistRemovals != null) {
+ this.blacklistRemovals.clear();
+ }
+ return;
+ }
+ initBlacklistRemovals();
+ this.blacklistRemovals.clear();
+ this.blacklistRemovals.addAll(resourceNames);
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3bf3b2e..0104817 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -207,6 +207,11 @@ message ResourceRequestProto {
optional bool relax_locality = 5 [default = true];
}
+message BlacklistRequestProto {
+ repeated string blacklist_additions = 1;
+ repeated string blacklist_removals = 2;
+}
+
////////////////////////////////////////////////////////////////////////
////// From client_RM_Protocol /////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index ed3f871..4a36967 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -54,8 +54,9 @@ message AllocateRequestProto {
optional ApplicationAttemptIdProto application_attempt_id = 1;
repeated ResourceRequestProto ask = 2;
repeated ContainerIdProto release = 3;
- optional int32 response_id = 4;
- optional float progress = 5;
+ optional BlacklistRequestProto blacklist_request = 4;
+ optional int32 response_id = 5;
+ optional float progress = 6;
}
message AllocateResponseProto {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 3094a93..d07cc38 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -69,6 +69,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidBlacklistRequestException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidResourceRequestException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -295,6 +296,14 @@ public AllocateResponse allocate(AllocateRequest request)
LOG.warn("Invalid resource ask by application " + appAttemptId, e);
throw RPCUtil.getRemoteException(e);
}
+
+ try {
+ SchedulerUtils.validateBlacklistRequest(request.getBlacklistRequest());
+ } catch (InvalidBlacklistRequestException e) {
+ LOG.warn("Invalid blacklist request by application " + appAttemptId, e);
+ throw RPCUtil.getRemoteException(e);
+ }
+
// Send new requests to appAttempt.
Allocation allocation =
this.rScheduler.allocate(appAttemptId, ask, release);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 6962a2c..506e8db 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -20,6 +20,7 @@
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -32,6 +33,7 @@
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -59,7 +61,8 @@
new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator());
final Map> requests =
new HashMap>();
-
+ final Set blacklist = new HashSet();
+
//private final ApplicationStore store;
private final ActiveUsersManager activeUsersManager;
@@ -118,8 +121,10 @@ public int getNewContainerId() {
* resources to be acquired
*/
synchronized public void updateResourceRequests(
- List requests) {
+ List requests,
+ BlacklistRequest blacklistRequest) {
QueueMetrics metrics = queue.getMetrics();
+
// Update resource requests
for (ResourceRequest request : requests) {
Priority priority = request.getPriority();
@@ -175,6 +180,21 @@ synchronized public void updateResourceRequests(
lastRequestContainers)));
}
}
+
+ // Update blacklist
+ if (blacklistRequest != null) {
+ // Add to blacklist
+ List plus = blacklistRequest.getBlacklistAdditions();
+ if (plus != null) {
+ blacklist.addAll(plus);
+ }
+
+ // Remove from blacklist
+ List minus = blacklistRequest.getBlacklistRemovals();
+ if (minus != null) {
+ blacklist.removeAll(minus);
+ }
+ }
}
synchronized public Collection getPriorities() {
@@ -197,6 +217,10 @@ public synchronized Resource getResource(Priority priority) {
return request.getCapability();
}
+ public synchronized boolean isBlacklisted(String resourceName) {
+ return blacklist.contains(resourceName);
+ }
+
/**
* Resources have been allocated to this application by the resource
* scheduler. Track them.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidBlacklistRequestException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidBlacklistRequestException.java
new file mode 100644
index 0000000..2087ebd
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidBlacklistRequestException.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.YarnException;
+
+/**
+ * The exception is thrown when the requested resource is out of the range
+ * of the configured lower and upper resource boundaries.
+ *
+ */
+public class InvalidBlacklistRequestException extends YarnException {
+
+ private static final long serialVersionUID = 384957911L;
+
+ public InvalidBlacklistRequestException(Throwable cause) {
+ super(cause);
+ }
+
+ public InvalidBlacklistRequestException(String message) {
+ super(message);
+ }
+
+ public InvalidBlacklistRequestException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java
index 3d1e7dd..85c17bb 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java
@@ -19,14 +19,16 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
/**
- * The exception is thrown when the requested resource is out of the range
- * of the configured lower and upper resource boundaries.
- *
+ * The exception is thrown when an application tries to blacklist
+ * {@link ResourceRequest#ANY}.
*/
public class InvalidResourceRequestException extends YarnException {
+ private static final long serialVersionUID = 13498237L;
+
public InvalidResourceRequestException(Throwable cause) {
super(cause);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c0a54c7..5d6b8ee 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -152,4 +153,14 @@ public static void validateResourceRequests(List ask,
}
}
+ public static void validateBlacklistRequest(BlacklistRequest blacklistRequest)
+ throws InvalidBlacklistRequestException {
+ if (blacklistRequest != null) {
+ List plus = blacklistRequest.getBlacklistAdditions();
+ if (plus != null && plus.contains(ResourceRequest.ANY)) {
+ throw new InvalidBlacklistRequestException(
+ "Cannot add " + ResourceRequest.ANY + " to the blacklist!");
+ }
+ }
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index aca2a12..4cc54ba 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -523,7 +523,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId,
application.showRequests();
// Update application requests
- application.updateResourceRequests(ask);
+ application.updateResourceRequests(ask, null);
LOG.debug("allocate: post-update");
application.showRequests();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 1d0cec2..a8a1acf 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -815,6 +815,11 @@ private synchronized FiCaSchedulerApp getApplication(
}
synchronized (application) {
+ // Check if this resource is on the blacklist
+ if (isBlacklisted(application, node)) {
+ continue;
+ }
+
// Schedule in priority order
for (Priority priority : application.getPriorities()) {
// Required resource
@@ -897,6 +902,28 @@ private synchronized FiCaSchedulerApp getApplication(
return NULL_ASSIGNMENT;
}
+
+ boolean isBlacklisted(FiCaSchedulerApp application, FiCaSchedulerNode node) {
+ if (application.isBlacklisted(node.getHostName())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping 'host' " + node.getHostName() +
+ " for " + application.getApplicationId() +
+ " since it has been blacklisted");
+ }
+ return true;
+ }
+
+ if (application.isBlacklisted(node.getRackName())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping 'rack' " + node.getRackName() +
+ " for " + application.getApplicationId() +
+ " since it has been blacklisted");
+ }
+ return true;
+ }
+
+ return false;
+ }
private synchronized CSAssignment
assignReservedContainer(FiCaSchedulerApp application,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index ba4efaa..68ed979 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -134,9 +135,9 @@ public String getUser() {
}
public synchronized void updateResourceRequests(
- List requests) {
+ List requests, BlacklistRequest blacklistRequest) {
if (!isStopped) {
- this.appSchedulingInfo.updateResourceRequests(requests);
+ this.appSchedulingInfo.updateResourceRequests(requests, blacklistRequest);
}
}
@@ -163,6 +164,10 @@ public synchronized int getTotalRequiredResources(Priority priority) {
public Resource getResource(Priority priority) {
return this.appSchedulingInfo.getResource(priority);
}
+
+ public boolean isBlacklisted(String resourceName) {
+ return this.appSchedulingInfo.isBlacklisted(resourceName);
+ }
/**
* Is this application pending?
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
index 9db15a6..f27789e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
@@ -138,7 +138,7 @@ public String getUser() {
public synchronized void updateResourceRequests(
List requests) {
- this.appSchedulingInfo.updateResourceRequests(requests);
+ this.appSchedulingInfo.updateResourceRequests(requests, null);
}
public Map getResourceRequests(Priority priority) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 2f4e70d..71eb7e6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -268,7 +268,7 @@ public Allocation allocate(
application.showRequests();
// Update application requests
- application.updateResourceRequests(ask);
+ application.updateResourceRequests(ask, null);
LOG.debug("allocate: post-update" +
" applicationId=" + applicationAttemptId +
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 84d9236..c69bebe 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -513,7 +513,7 @@ public void testHeadroom() throws Exception {
app_0_0_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_0_0.updateResourceRequests(app_0_0_requests);
+ app_0_0.updateResourceRequests(app_0_0_requests, null);
// Schedule to compute
queue.assignContainers(clusterResource, node_0);
@@ -532,7 +532,7 @@ public void testHeadroom() throws Exception {
app_0_1_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_0_1.updateResourceRequests(app_0_1_requests);
+ app_0_1.updateResourceRequests(app_0_1_requests, null);
// Schedule to compute
queue.assignContainers(clusterResource, node_0); // Schedule to compute
@@ -551,7 +551,7 @@ public void testHeadroom() throws Exception {
app_1_0_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_1_0.updateResourceRequests(app_1_0_requests);
+ app_1_0.updateResourceRequests(app_1_0_requests, null);
// Schedule to compute
queue.assignContainers(clusterResource, node_0); // Schedule to compute
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 908e9a8..681bf01 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -43,6 +43,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -293,7 +294,7 @@ public void testSingleQueueOneUserMetrics() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
// Start testing...
@@ -415,11 +416,11 @@ public void testSingleQueueWithOneUser() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
// Start testing...
@@ -548,11 +549,11 @@ public void testUserLimits() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
/**
* Start testing...
@@ -641,11 +642,11 @@ public void testHeadroomWithMaxCap() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
/**
* Start testing...
@@ -680,7 +681,7 @@ public void testHeadroomWithMaxCap() throws Exception {
a.setMaxCapacity(.1f);
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
assertEquals(2, a.getActiveUsersManager().getNumActiveUsers());
// No more to user_0 since he is already over user-limit
@@ -697,7 +698,7 @@ public void testHeadroomWithMaxCap() throws Exception {
LOG.info("here");
app_1.updateResourceRequests(Collections.singletonList( // unset
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
assertEquals(1, a.getActiveUsersManager().getNumActiveUsers());
a.assignContainers(clusterResource, node_1);
assertEquals(1*GB, app_2.getHeadroom().getMemory()); // hit queue max-cap
@@ -758,11 +759,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
/**
* Start testing...
@@ -792,11 +793,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception {
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
app_3.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
// Now allocations should goto app_2 since
// user_0 is at limit inspite of high user-limit-factor
@@ -920,11 +921,11 @@ public void testReservation() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
// Start testing...
@@ -1022,7 +1023,7 @@ public void testStolenReservedContainer() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
// Setup app_1 to request a 4GB container on host_0 and
// another 4GB container anywhere.
@@ -1034,7 +1035,7 @@ public void testStolenReservedContainer() throws Exception {
true, priority, recordFactory));
appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2,
true, priority, recordFactory));
- app_1.updateResourceRequests(appRequests_1);
+ app_1.updateResourceRequests(appRequests_1, null);
// Start testing...
@@ -1128,11 +1129,11 @@ public void testReservationExchange() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null);
// Start testing...
@@ -1255,7 +1256,7 @@ public void testLocalityScheduling() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null);
// Start testing...
CSAssignment assignment = null;
@@ -1320,7 +1321,7 @@ public void testLocalityScheduling() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null);
assertEquals(2, app_0.getTotalRequiredResources(priority));
String host_3 = "127.0.0.4"; // on rack_1
@@ -1411,7 +1412,7 @@ public void testApplicationPriorityScheduling() throws Exception {
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1,
true, priority_2, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null);
// Start testing...
@@ -1526,7 +1527,7 @@ public void testSchedulingConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(rack_1, 1*GB, 1,
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null);
// Start testing...
@@ -1535,7 +1536,7 @@ public void testSchedulingConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null);
// NODE_LOCAL - node_0_1
a.assignContainers(clusterResource, node_0_0);
@@ -1558,7 +1559,7 @@ public void testSchedulingConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null);
// No allocation on node_0_1 even though it's node/rack local since
// required(rack_1) == 0
@@ -1759,7 +1760,6 @@ public void testLocalityConstraints() throws Exception {
// Setup some nodes and racks
String host_0_0 = "127.0.0.1";
String rack_0 = "rack_0";
- FiCaSchedulerNode node_0_0 = TestUtils.getMockNode(host_0_0, rack_0, 0, 8*GB);
String host_0_1 = "127.0.0.2";
FiCaSchedulerNode node_0_1 = TestUtils.getMockNode(host_0_1, rack_0, 0, 8*GB);
@@ -1775,6 +1775,10 @@ public void testLocalityConstraints() throws Exception {
numNodes * (8*GB), numNodes * 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ // Blacklist
+ BlacklistRequest blacklistRequest =
+ BlacklistRequest.newInstance(null, null);
+
// Setup resource-requests
// resourceName:
// host_0_0: < 1, 1GB, 1, true >
@@ -1789,6 +1793,7 @@ public void testLocalityConstraints() throws Exception {
// host_0_1: 8G
// host_1_0: 8G
// host_1_1: 8G
+ // Blacklist:
Priority priority = TestUtils.createMockPriority(1);
List app_0_requests_0 = new ArrayList();
app_0_requests_0.add(
@@ -1803,8 +1808,10 @@ public void testLocalityConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
false, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ blacklistRequest.setBlacklistAdditions(Collections.singletonList(host_0_0));
+ app_0.updateResourceRequests(app_0_requests_0, blacklistRequest);
app_0_requests_0.clear();
+ blacklistRequest.setBlacklistAdditions(null); // reset
//
// Start testing...
@@ -1838,12 +1845,14 @@ public void testLocalityConstraints() throws Exception {
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0
- // Allow rack-locality for rack_1
+ // Allow rack-locality for rack_1, but blacklist node_1_1
app_0_requests_0.add(
TestUtils.createResourceRequest(rack_1, 1*GB, 1,
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ blacklistRequest.setBlacklistAdditions(Collections.singletonList(host_1_1));
+ app_0.updateResourceRequests(app_0_requests_0, blacklistRequest);
app_0_requests_0.clear();
+ blacklistRequest.setBlacklistAdditions(null); // reset
// resourceName:
// host_0_0: < 1, 1GB, 1, true >
@@ -1860,6 +1869,33 @@ public void testLocalityConstraints() throws Exception {
// host_1_1: 8G
// node_1_1
+ // Shouldn't allocate since node_1_1 is blacklisted
+ a.assignContainers(clusterResource, node_1_1);
+ verify(app_0, never()).allocate(any(NodeType.class), eq(node_0_1),
+ any(Priority.class), any(ResourceRequest.class), any(Container.class));
+ assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0
+
+ // Now, remove node_1_1 from blacklist, but add rack_1 to blacklist
+ blacklistRequest.setBlacklistAdditions(Collections.singletonList(rack_1));
+ blacklistRequest.setBlacklistRemovals(Collections.singletonList(host_1_1));
+ app_0.updateResourceRequests(app_0_requests_0, blacklistRequest);
+ app_0_requests_0.clear();
+ blacklistRequest.setBlacklistAdditions(null);
+ blacklistRequest.setBlacklistRemovals(null);
+
+ // node_1_1
+ // Shouldn't allocate since rack_1 is blacklisted
+ a.assignContainers(clusterResource, node_1_1);
+ verify(app_0, never()).allocate(any(NodeType.class), eq(node_0_1),
+ any(Priority.class), any(ResourceRequest.class), any(Container.class));
+ assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0
+
+ // Now clear blacklist
+ blacklistRequest.setBlacklistRemovals(Collections.singletonList(rack_1));
+ app_0.updateResourceRequests(app_0_requests_0, blacklistRequest);
+ app_0_requests_0.clear();
+ blacklistRequest.setBlacklistRemovals(null);
+
// Now, should allocate since RR(rack_1) = relax: true
a.assignContainers(clusterResource, node_1_1);
verify(app_0).allocate(eq(NodeType.RACK_LOCAL), eq(node_1_1),
@@ -1874,7 +1910,7 @@ public void testLocalityConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
false, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null);
app_0_requests_0.clear();
// resourceName: