diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
index 63bd7b3..c325940 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
@@ -1214,7 +1214,8 @@ public MyFifoScheduler(RMContext rmContext) {
@Override
public synchronized Allocation allocate(
ApplicationAttemptId applicationAttemptId, List ask,
- List release) {
+ List release,
+ List blacklistAdditions, List blacklistRemovals) {
List askCopy = new ArrayList();
for (ResourceRequest req : ask) {
ResourceRequest reqCopy = ResourceRequest.newInstance(req
@@ -1223,7 +1224,9 @@ public synchronized Allocation allocate(
askCopy.add(reqCopy);
}
lastAsk = ask;
- return super.allocate(applicationAttemptId, askCopy, release);
+ return super.allocate(
+ applicationAttemptId, askCopy, release,
+ blacklistAdditions, blacklistRemovals);
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java
index dedd9a9..b5f188d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
/**
* The protocol between a live instance of ApplicationMaster
@@ -60,6 +61,8 @@
* @return registration respose
* @throws YarnException
* @throws IOException
+ * @see RegisterApplicationMasterRequest
+ * @see RegisterApplicationMasterResponse
*/
public RegisterApplicationMasterResponse registerApplicationMaster(
RegisterApplicationMasterRequest request)
@@ -80,6 +83,8 @@ public RegisterApplicationMasterResponse registerApplicationMaster(
* @return completion response
* @throws YarnException
* @throws IOException
+ * @see FinishApplicationMasterRequest
+ * @see FinishApplicationMasterResponse
*/
public FinishApplicationMasterResponse finishApplicationMaster(
FinishApplicationMasterRequest request)
@@ -91,12 +96,16 @@ public FinishApplicationMasterResponse finishApplicationMaster(
*
*
The ApplicationMaster uses this interface to provide a list
* of {@link ResourceRequest} and returns unused {@link Container} allocated
- * to it via {@link AllocateRequest}.
+ * to it via {@link AllocateRequest}. Optionally, the
+ * ApplicationMaster can also blacklist resources
+ * which it doesn't want to use.
*
* This also doubles up as a heartbeat to let the
* ResourceManager know that the ApplicationMaster
* is alive. Thus, applications should periodically make this call to be kept
- * alive. The frequency depends on ??
+ * alive. The frequency depends on
+ * {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to
+ * {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.
*
* The ResourceManager responds with list of allocated
* {@link Container}, status of completed containers and headroom information
@@ -110,6 +119,8 @@ public FinishApplicationMasterResponse finishApplicationMaster(
* @return allocation response
* @throws YarnException
* @throws IOException
+ * @see AllocateRequest
+ * @see AllocateResponse
*/
public AllocateResponse allocate(AllocateRequest request)
throws YarnException, IOException;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index 4767922..76357a5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.AMRMProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -127,6 +128,7 @@ public static AllocateRequest newInstance(
* Get the list of ResourceRequest to update the
* ResourceManager about the application's resource requirements.
* @return the list of ResourceRequest
+ * @see ResourceRequest
*/
@Public
@Stable
@@ -138,6 +140,7 @@ public static AllocateRequest newInstance(
* @param resourceRequests list of ResourceRequest to update the
* ResourceManager about the application's
* resource requirements
+ * @see ResourceRequest
*/
@Public
@Stable
@@ -157,10 +160,36 @@ public static AllocateRequest newInstance(
* Set the list of ContainerId of containers being
* released by the ApplicationMaster
* @param releaseContainers list of ContainerId of
- * containers being released by the <
- * code>ApplicationMaster
+ * containers being released by the
+ * ApplicationMaster
*/
@Public
@Stable
public abstract void setReleaseList(List releaseContainers);
+
+ /**
+ * Get the BlacklistRequest being sent by the
+ * ApplicationMaster.
+ * @return the BlacklistRequest being sent by the
+ * ApplicationMaster
+ * @see BlacklistRequest
+ */
+ @Public
+ @Stable
+ public abstract BlacklistRequest getBlacklistRequest();
+
+ /**
+ * Set the BlacklistRequest to inform the
+ * ResourceManager about the blacklist additions and removals
+ * per the ApplicationMaster.
+ *
+ * @param blacklistRequest the BlacklistRequest to inform the
+ * ResourceManager about the
+ * blacklist additions and removals
+ * per the ApplicationMaster
+ * @see BlacklistRequest
+ */
+ @Public
+ @Stable
+ public abstract void setBlacklistRequest(BlacklistRequest blacklistRequest);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
index 2a78efa..1feae0e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
@@ -25,12 +25,15 @@
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.BlacklistRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.BlacklistRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto;
@@ -46,6 +49,7 @@
private ApplicationAttemptId applicationAttemptID = null;
private List ask = null;
private List release = null;
+ private BlacklistRequest blacklistRequest = null;
public AllocateRequestPBImpl() {
@@ -94,6 +98,9 @@ private void mergeLocalToBuilder() {
if (this.release != null) {
addReleasesToProto();
}
+ if (this.blacklistRequest != null) {
+ builder.setBlacklistRequest(convertToProtoFormat(this.blacklistRequest));
+ }
}
private void mergeLocalToProto() {
@@ -161,6 +168,7 @@ public void setProgress(float progress) {
initAsks();
return this.ask;
}
+
@Override
public void setAskList(final List resourceRequests) {
if(resourceRequests == null) {
@@ -171,6 +179,28 @@ public void setAskList(final List resourceRequests) {
this.ask.addAll(resourceRequests);
}
+ @Override
+ public BlacklistRequest getBlacklistRequest() {
+ AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.blacklistRequest != null) {
+ return this.blacklistRequest;
+ }
+ if (!p.hasBlacklistRequest()) {
+ return null;
+ }
+ this.blacklistRequest = convertFromProtoFormat(p.getBlacklistRequest());
+ return this.blacklistRequest;
+ }
+
+ @Override
+ public void setBlacklistRequest(BlacklistRequest blacklistRequest) {
+ maybeInitBuilder();
+ if (blacklistRequest == null) {
+ builder.clearBlacklistRequest();
+ }
+ this.blacklistRequest = blacklistRequest;
+ }
+
private void initAsks() {
if (this.ask != null) {
return;
@@ -302,4 +332,14 @@ private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
private ContainerIdProto convertToProtoFormat(ContainerId t) {
return ((ContainerIdPBImpl)t).getProto();
}
+
+ private BlacklistRequestPBImpl convertFromProtoFormat(BlacklistRequestProto p) {
+ return new BlacklistRequestPBImpl(p);
+ }
+
+ private BlacklistRequestProto convertToProtoFormat(BlacklistRequest t) {
+ return ((BlacklistRequestPBImpl)t).getProto();
+ }
+
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BlacklistRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BlacklistRequest.java
new file mode 100644
index 0000000..897ab62
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BlacklistRequest.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@link BlacklistRequest} encapsulates the list of resource-names which
+ * should be added or removed from the blacklist of resources for the
+ * application.
+ *
+ * @see ResourceRequest
+ * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ */
+@Public
+@Stable
+public abstract class BlacklistRequest {
+
+ public static BlacklistRequest newInstance(
+ List additions, List removals) {
+ BlacklistRequest blacklistRequest =
+ Records.newRecord(BlacklistRequest.class);
+ blacklistRequest.setBlacklistAdditions(additions);
+ blacklistRequest.setBlacklistRemovals(removals);
+ return blacklistRequest;
+ }
+
+ /**
+ * Get the list of resources which should be added to the
+ * application blacklist.
+ *
+ * @return list of resources which should be added to the
+ * application blacklist
+ */
+ public abstract List getBlacklistAdditions();
+
+ /**
+ * Set list of resources which should be added to the application blacklist.
+ *
+ * @param resourceNames list of resources which should be added to the
+ * application blacklist
+ */
+ public abstract void setBlacklistAdditions(List resourceNames);
+
+ /**
+ * Get the list of resources which should be removed from the
+ * application blacklist.
+ *
+ * @return list of resources which should be removed from the
+ * application blacklist
+ */
+ public abstract List getBlacklistRemovals();
+
+ /**
+ * Set list of resources which should be removed from the
+ * application blacklist.
+ *
+ * @param resourceNames list of resources which should be removed from the
+ * application blacklist
+ */
+ public abstract void setBlacklistRemovals(List resourceNames);
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index d4afda8..7f119fd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -40,9 +40,15 @@
*
* {@link Resource} required for each request.
*
- * Number of containers of such specifications which are required
+ * Number of containers, of above specifications, which are required
* by the application.
*
+ *
+ * A boolean relaxLocality flag, defaulting to true,
+ * which tells the ResourceManager if the application wants
+ * locality to be loose (i.e. allows fall-through to rack or any)
+ * or strict (i.e. specify hard constraint on resource allocation).
+ *
*
*
*
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/BlacklistRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/BlacklistRequestPBImpl.java
new file mode 100644
index 0000000..736b205
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/BlacklistRequestPBImpl.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
+import org.apache.hadoop.yarn.proto.YarnProtos.BlacklistRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.BlacklistRequestProtoOrBuilder;
+
+public class BlacklistRequestPBImpl extends BlacklistRequest {
+
+ BlacklistRequestProto proto = null;
+ BlacklistRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ List blacklistAdditions = null;
+ List blacklistRemovals = null;
+
+ public BlacklistRequestPBImpl() {
+ builder = BlacklistRequestProto.newBuilder();
+ }
+
+ public BlacklistRequestPBImpl(BlacklistRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public BlacklistRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = BlacklistRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.blacklistAdditions != null) {
+ addBlacklistAdditionsToProto();
+ }
+ if (this.blacklistRemovals != null) {
+ addBlacklistRemovalsToProto();
+ }
+ }
+
+ private void addBlacklistAdditionsToProto() {
+ maybeInitBuilder();
+ builder.clearBlacklistAdditions();
+ if (this.blacklistAdditions == null) {
+ return;
+ }
+ builder.addAllBlacklistAdditions(this.blacklistAdditions);
+ }
+
+ private void addBlacklistRemovalsToProto() {
+ maybeInitBuilder();
+ builder.clearBlacklistAdditions();
+ if (this.blacklistRemovals == null) {
+ return;
+ }
+ builder.addAllBlacklistRemovals(this.blacklistRemovals);
+ }
+
+ private void initBlacklistAdditions() {
+ if (this.blacklistAdditions != null) {
+ return;
+ }
+ BlacklistRequestProtoOrBuilder p = viaProto ? proto : builder;
+ List list = p.getBlacklistAdditionsList();
+ this.blacklistAdditions = new ArrayList();
+ this.blacklistAdditions.addAll(list);
+ }
+
+ private void initBlacklistRemovals() {
+ if (this.blacklistRemovals != null) {
+ return;
+ }
+ BlacklistRequestProtoOrBuilder p = viaProto ? proto : builder;
+ List list = p.getBlacklistRemovalsList();
+ this.blacklistRemovals = new ArrayList();
+ this.blacklistRemovals.addAll(list);
+ }
+
+ @Override
+ public List getBlacklistAdditions() {
+ initBlacklistAdditions();
+ return this.blacklistAdditions;
+ }
+
+ @Override
+ public void setBlacklistAdditions(List resourceNames) {
+ if (resourceNames == null) {
+ if (this.blacklistAdditions != null) {
+ this.blacklistAdditions.clear();
+ }
+ return;
+ }
+ initBlacklistAdditions();
+ this.blacklistAdditions.clear();
+ this.blacklistAdditions.addAll(resourceNames);
+ }
+
+ @Override
+ public List getBlacklistRemovals() {
+ initBlacklistRemovals();
+ return this.blacklistRemovals;
+ }
+
+ @Override
+ public void setBlacklistRemovals(List resourceNames) {
+ if (resourceNames == null) {
+ if (this.blacklistRemovals != null) {
+ this.blacklistRemovals.clear();
+ }
+ return;
+ }
+ initBlacklistRemovals();
+ this.blacklistRemovals.clear();
+ this.blacklistRemovals.addAll(resourceNames);
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3bf3b2e..0104817 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -207,6 +207,11 @@ message ResourceRequestProto {
optional bool relax_locality = 5 [default = true];
}
+message BlacklistRequestProto {
+ repeated string blacklist_additions = 1;
+ repeated string blacklist_removals = 2;
+}
+
////////////////////////////////////////////////////////////////////////
////// From client_RM_Protocol /////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index b00e513..3dbacfd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -54,8 +54,9 @@ message AllocateRequestProto {
optional ApplicationAttemptIdProto application_attempt_id = 1;
repeated ResourceRequestProto ask = 2;
repeated ContainerIdProto release = 3;
- optional int32 response_id = 4;
- optional float progress = 5;
+ optional BlacklistRequestProto blacklist_request = 4;
+ optional int32 response_id = 5;
+ optional float progress = 6;
}
message AllocateResponseProto {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 53790bf..96a4d93 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -69,6 +69,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidBlacklistRequestException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidResourceRequestException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -295,9 +296,19 @@ public AllocateResponse allocate(AllocateRequest request)
LOG.warn("Invalid resource ask by application " + appAttemptId, e);
throw RPCUtil.getRemoteException(e);
}
+
+ try {
+ SchedulerUtils.validateBlacklistRequest(request.getBlacklistRequest());
+ } catch (InvalidBlacklistRequestException e) {
+ LOG.warn("Invalid blacklist request by application " + appAttemptId, e);
+ throw RPCUtil.getRemoteException(e);
+ }
+
// Send new requests to appAttempt.
Allocation allocation =
- this.rScheduler.allocate(appAttemptId, ask, release);
+ this.rScheduler.allocate(appAttemptId, ask, release,
+ request.getBlacklistRequest().getBlacklistAdditions(),
+ request.getBlacklistRequest().getBlacklistRemovals());
RMApp app = this.rmContext.getRMApps().get(
appAttemptId.getApplicationId());
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 69f058a..bb2e252 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -803,7 +803,7 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
// AM resource has been checked when submission
Allocation amContainerAllocation = appAttempt.scheduler.allocate(
appAttempt.applicationAttemptId,
- Collections.singletonList(request), EMPTY_CONTAINER_RELEASE_LIST);
+ Collections.singletonList(request), EMPTY_CONTAINER_RELEASE_LIST, null, null);
if (amContainerAllocation != null
&& amContainerAllocation.getContainers() != null) {
assert (amContainerAllocation.getContainers().size() == 0);
@@ -827,7 +827,7 @@ public void transition(RMAppAttemptImpl appAttempt,
// Acquire the AM container from the scheduler.
Allocation amContainerAllocation = appAttempt.scheduler.allocate(
appAttempt.applicationAttemptId, EMPTY_CONTAINER_REQUEST_LIST,
- EMPTY_CONTAINER_RELEASE_LIST);
+ EMPTY_CONTAINER_RELEASE_LIST, null, null);
// Set the masterContainer
appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 6962a2c..416e1e6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -20,6 +20,7 @@
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -59,7 +60,8 @@
new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator());
final Map> requests =
new HashMap>();
-
+ final Set blacklist = new HashSet();
+
//private final ApplicationStore store;
private final ActiveUsersManager activeUsersManager;
@@ -114,12 +116,15 @@ public int getNewContainerId() {
* application, by asking for more resources and releasing resources acquired
* by the application.
*
- * @param requests
- * resources to be acquired
+ * @param requests resources to be acquired
+ * @param blacklistAdditions resources to be added to the blacklist
+ * @param blacklistRemovals resources to be removed from the blacklist
*/
synchronized public void updateResourceRequests(
- List requests) {
+ List requests,
+ List blacklistAdditions, List blacklistRemovals) {
QueueMetrics metrics = queue.getMetrics();
+
// Update resource requests
for (ResourceRequest request : requests) {
Priority priority = request.getPriority();
@@ -175,6 +180,20 @@ synchronized public void updateResourceRequests(
lastRequestContainers)));
}
}
+
+ //
+ // Update blacklist
+ //
+
+ // Add to blacklist
+ if (blacklistAdditions != null) {
+ blacklist.addAll(blacklistAdditions);
+ }
+
+ // Remove from blacklist
+ if (blacklistRemovals != null) {
+ blacklist.removeAll(blacklistRemovals);
+ }
}
synchronized public Collection getPriorities() {
@@ -197,6 +216,10 @@ public synchronized Resource getResource(Priority priority) {
return request.getCapability();
}
+ public synchronized boolean isBlacklisted(String resourceName) {
+ return blacklist.contains(resourceName);
+ }
+
/**
* Resources have been allocated to this application by the resource
* scheduler. Track them.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidBlacklistRequestException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidBlacklistRequestException.java
new file mode 100644
index 0000000..604c220
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidBlacklistRequestException.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.YarnRuntimeException;
+
+/**
+ * The exception is thrown when an application tries to blacklist
+ * {@link ResourceRequest#ANY}.
+ */
+public class InvalidBlacklistRequestException extends YarnRuntimeException {
+
+ private static final long serialVersionUID = 384957911L;
+
+ public InvalidBlacklistRequestException(Throwable cause) {
+ super(cause);
+ }
+
+ public InvalidBlacklistRequestException(String message) {
+ super(message);
+ }
+
+ public InvalidBlacklistRequestException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java
index 27628f37..5de8b11 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java
@@ -19,14 +19,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.yarn.YarnRuntimeException;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
/**
- * The exception is thrown when the requested resource is out of the range
+ * The exception is thrown when a {@link ResourceRequest} is out of the range
* of the configured lower and upper resource boundaries.
*
*/
public class InvalidResourceRequestException extends YarnRuntimeException {
+ private static final long serialVersionUID = 13498237L;
+
public InvalidResourceRequestException(Throwable cause) {
super(cause);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c0a54c7..5d6b8ee 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -152,4 +153,14 @@ public static void validateResourceRequests(List ask,
}
}
+ public static void validateBlacklistRequest(BlacklistRequest blacklistRequest)
+ throws InvalidBlacklistRequestException {
+ if (blacklistRequest != null) {
+ List plus = blacklistRequest.getBlacklistAdditions();
+ if (plus != null && plus.contains(ResourceRequest.ANY)) {
+ throw new InvalidBlacklistRequestException(
+ "Cannot add " + ResourceRequest.ANY + " to the blacklist!");
+ }
+ }
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index f084649..08f667c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -95,6 +95,8 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues,
* @param appAttemptId
* @param ask
* @param release
+ * @param blacklistAdditions
+ * @param blacklistRemovals
* @return the {@link Allocation} for the application
*/
@Public
@@ -102,7 +104,9 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues,
Allocation
allocate(ApplicationAttemptId appAttemptId,
List ask,
- List release);
+ List release,
+ List blacklistAdditions,
+ List blacklistRemovals);
/**
* Get node resource usage report.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index aca2a12..4ebf94b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -472,7 +472,8 @@ private synchronized void doneApplication(
@Override
@Lock(Lock.NoLock.class)
public Allocation allocate(ApplicationAttemptId applicationAttemptId,
- List ask, List release) {
+ List ask, List release,
+ List blacklistAdditions, List blacklistRemovals) {
FiCaSchedulerApp application = getApplication(applicationAttemptId);
if (application == null) {
@@ -523,7 +524,8 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId,
application.showRequests();
// Update application requests
- application.updateResourceRequests(ask);
+ application.updateResourceRequests(ask,
+ blacklistAdditions, blacklistRemovals);
LOG.debug("allocate: post-update");
application.showRequests();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 129d15c..130d8b6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -815,6 +815,11 @@ private synchronized FiCaSchedulerApp getApplication(
}
synchronized (application) {
+ // Check if this resource is on the blacklist
+ if (isBlacklisted(application, node)) {
+ continue;
+ }
+
// Schedule in priority order
for (Priority priority : application.getPriorities()) {
// Required resource
@@ -897,6 +902,28 @@ private synchronized FiCaSchedulerApp getApplication(
return NULL_ASSIGNMENT;
}
+
+ boolean isBlacklisted(FiCaSchedulerApp application, FiCaSchedulerNode node) {
+ if (application.isBlacklisted(node.getHostName())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping 'host' " + node.getHostName() +
+ " for " + application.getApplicationId() +
+ " since it has been blacklisted");
+ }
+ return true;
+ }
+
+ if (application.isBlacklisted(node.getRackName())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping 'rack' " + node.getRackName() +
+ " for " + application.getApplicationId() +
+ " since it has been blacklisted");
+ }
+ return true;
+ }
+
+ return false;
+ }
private synchronized CSAssignment
assignReservedContainer(FiCaSchedulerApp application,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index ba4efaa..86bed0d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -134,9 +134,11 @@ public String getUser() {
}
public synchronized void updateResourceRequests(
- List requests) {
+ List requests,
+ List blacklistAdditions, List blacklistRemovals) {
if (!isStopped) {
- this.appSchedulingInfo.updateResourceRequests(requests);
+ this.appSchedulingInfo.updateResourceRequests(requests,
+ blacklistAdditions, blacklistRemovals);
}
}
@@ -163,6 +165,10 @@ public synchronized int getTotalRequiredResources(Priority priority) {
public Resource getResource(Priority priority) {
return this.appSchedulingInfo.getResource(priority);
}
+
+ public boolean isBlacklisted(String resourceName) {
+ return this.appSchedulingInfo.isBlacklisted(resourceName);
+ }
/**
* Is this application pending?
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
index 9db15a6..4aaf74b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
@@ -138,7 +138,7 @@ public String getUser() {
public synchronized void updateResourceRequests(
List requests) {
- this.appSchedulingInfo.updateResourceRequests(requests);
+ this.appSchedulingInfo.updateResourceRequests(requests, null, null);
}
public Map getResourceRequests(Priority priority) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index f8cff0e..b996952 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -718,7 +718,7 @@ private synchronized void removeNode(RMNode rmNode) {
@Override
public Allocation allocate(ApplicationAttemptId appAttemptId,
- List ask, List release) {
+ List ask, List release, List blacklistAdditions, List blacklistRemovals) {
// Make sure this application exists
FSSchedulerApp application = applications.get(appAttemptId);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 0de8c59..e7a2e94 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -222,7 +222,7 @@ public Resource getMaximumResourceCapability() {
@Override
public Allocation allocate(
ApplicationAttemptId applicationAttemptId, List ask,
- List release) {
+ List release, List blacklistAdditions, List blacklistRemovals) {
FiCaSchedulerApp application = getApplication(applicationAttemptId);
if (application == null) {
LOG.error("Calling allocate on removed " +
@@ -268,7 +268,7 @@ public Allocation allocate(
application.showRequests();
// Update application requests
- application.updateResourceRequests(ask);
+ application.updateResourceRequests(ask, null, null);
LOG.debug("allocate: post-update" +
" applicationId=" + applicationAttemptId +
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
index af626aa..56fc119 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
@@ -265,7 +265,7 @@ private synchronized void addResourceRequest(
// Get resources from the ResourceManager
resourceManager.getResourceScheduler().allocate(applicationAttemptId,
- new ArrayList(ask), new ArrayList());
+ new ArrayList(ask), new ArrayList(), null, null);
System.out.println("-=======" + applicationAttemptId);
System.out.println("----------" + resourceManager.getRMContext().getRMApps()
.get(applicationId).getRMAppAttempt(applicationAttemptId));
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
index aa5e286..160d8d4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
@@ -273,23 +273,23 @@ public void testHeadroom() throws Exception {
List ask1 = new ArrayList();
ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
- fs.allocate(appAttemptId1, ask1, emptyId);
+ fs.allocate(appAttemptId1, ask1, emptyId, null, null);
// Ask for a 2 GB container for app 2
List ask2 = new ArrayList();
ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
- fs.allocate(appAttemptId2, ask2, emptyId);
+ fs.allocate(appAttemptId2, ask2, emptyId, null, null);
// Trigger container assignment
fs.handle(new NodeUpdateSchedulerEvent(n1));
// Get the allocation for the applications and verify headroom
- Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId);
+ Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
Assert.assertEquals("Allocation headroom", 1 * GB,
allocation1.getResourceLimit().getMemory());
- Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId);
+ Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null);
Assert.assertEquals("Allocation headroom", 1 * GB,
allocation2.getResourceLimit().getMemory());
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index f3ce664..50baaa0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -330,7 +330,7 @@ private void testAppAttemptScheduledState() {
applicationAttempt.getAppAttemptState());
verify(scheduler, times(expectedAllocateCount)).
allocate(any(ApplicationAttemptId.class),
- any(List.class), any(List.class));
+ any(List.class), any(List.class), null, null);
assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
assertNull(applicationAttempt.getMasterContainer());
@@ -354,7 +354,7 @@ private void testAppAttemptAllocatedState(Container amContainer) {
verify(applicationMasterLauncher).handle(any(AMLauncherEvent.class));
verify(scheduler, times(2)).
allocate(
- any(ApplicationAttemptId.class), any(List.class), any(List.class));
+ any(ApplicationAttemptId.class), any(List.class), any(List.class), null, null);
}
/**
@@ -481,7 +481,7 @@ private Container allocateApplicationAttempt() {
scheduler.allocate(
any(ApplicationAttemptId.class),
any(List.class),
- any(List.class))).
+ any(List.class), null, null)).
thenReturn(allocation);
applicationAttempt.handle(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 84d9236..f8dd885 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -513,7 +513,7 @@ public void testHeadroom() throws Exception {
app_0_0_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_0_0.updateResourceRequests(app_0_0_requests);
+ app_0_0.updateResourceRequests(app_0_0_requests, null, null);
// Schedule to compute
queue.assignContainers(clusterResource, node_0);
@@ -532,7 +532,7 @@ public void testHeadroom() throws Exception {
app_0_1_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_0_1.updateResourceRequests(app_0_1_requests);
+ app_0_1.updateResourceRequests(app_0_1_requests, null, null);
// Schedule to compute
queue.assignContainers(clusterResource, node_0); // Schedule to compute
@@ -551,7 +551,7 @@ public void testHeadroom() throws Exception {
app_1_0_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_1_0.updateResourceRequests(app_1_0_requests);
+ app_1_0.updateResourceRequests(app_1_0_requests, null, null);
// Schedule to compute
queue.assignContainers(clusterResource, node_0); // Schedule to compute
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 908e9a8..a0bded2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -43,6 +43,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.BlacklistRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -293,7 +294,7 @@ public void testSingleQueueOneUserMetrics() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
// Start testing...
@@ -415,11 +416,11 @@ public void testSingleQueueWithOneUser() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
// Start testing...
@@ -548,11 +549,11 @@ public void testUserLimits() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
/**
* Start testing...
@@ -641,11 +642,11 @@ public void testHeadroomWithMaxCap() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
/**
* Start testing...
@@ -680,7 +681,7 @@ public void testHeadroomWithMaxCap() throws Exception {
a.setMaxCapacity(.1f);
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
assertEquals(2, a.getActiveUsersManager().getNumActiveUsers());
// No more to user_0 since he is already over user-limit
@@ -697,7 +698,7 @@ public void testHeadroomWithMaxCap() throws Exception {
LOG.info("here");
app_1.updateResourceRequests(Collections.singletonList( // unset
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
assertEquals(1, a.getActiveUsersManager().getNumActiveUsers());
a.assignContainers(clusterResource, node_1);
assertEquals(1*GB, app_2.getHeadroom().getMemory()); // hit queue max-cap
@@ -758,11 +759,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
/**
* Start testing...
@@ -792,11 +793,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception {
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
app_3.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
// Now allocations should goto app_2 since
// user_0 is at limit inspite of high user-limit-factor
@@ -920,11 +921,11 @@ public void testReservation() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
// Start testing...
@@ -1022,7 +1023,7 @@ public void testStolenReservedContainer() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
// Setup app_1 to request a 4GB container on host_0 and
// another 4GB container anywhere.
@@ -1034,7 +1035,7 @@ public void testStolenReservedContainer() throws Exception {
true, priority, recordFactory));
appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2,
true, priority, recordFactory));
- app_1.updateResourceRequests(appRequests_1);
+ app_1.updateResourceRequests(appRequests_1, null, null);
// Start testing...
@@ -1128,11 +1129,11 @@ public void testReservationExchange() throws Exception {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true,
- priority, recordFactory)));
+ priority, recordFactory)), null, null);
// Start testing...
@@ -1255,7 +1256,7 @@ public void testLocalityScheduling() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null, null);
// Start testing...
CSAssignment assignment = null;
@@ -1320,7 +1321,7 @@ public void testLocalityScheduling() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null, null);
assertEquals(2, app_0.getTotalRequiredResources(priority));
String host_3 = "127.0.0.4"; // on rack_1
@@ -1411,7 +1412,7 @@ public void testApplicationPriorityScheduling() throws Exception {
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1,
true, priority_2, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null, null);
// Start testing...
@@ -1526,7 +1527,7 @@ public void testSchedulingConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(rack_1, 1*GB, 1,
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null, null);
// Start testing...
@@ -1535,7 +1536,7 @@ public void testSchedulingConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null, null);
// NODE_LOCAL - node_0_1
a.assignContainers(clusterResource, node_0_0);
@@ -1558,7 +1559,7 @@ public void testSchedulingConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null, null);
// No allocation on node_0_1 even though it's node/rack local since
// required(rack_1) == 0
@@ -1759,7 +1760,6 @@ public void testLocalityConstraints() throws Exception {
// Setup some nodes and racks
String host_0_0 = "127.0.0.1";
String rack_0 = "rack_0";
- FiCaSchedulerNode node_0_0 = TestUtils.getMockNode(host_0_0, rack_0, 0, 8*GB);
String host_0_1 = "127.0.0.2";
FiCaSchedulerNode node_0_1 = TestUtils.getMockNode(host_0_1, rack_0, 0, 8*GB);
@@ -1789,6 +1789,7 @@ public void testLocalityConstraints() throws Exception {
// host_0_1: 8G
// host_1_0: 8G
// host_1_1: 8G
+ // Blacklist:
Priority priority = TestUtils.createMockPriority(1);
List app_0_requests_0 = new ArrayList();
app_0_requests_0.add(
@@ -1803,7 +1804,8 @@ public void testLocalityConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
false, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0,
+ Collections.singletonList(host_0_0), null);
app_0_requests_0.clear();
//
@@ -1830,6 +1832,7 @@ public void testLocalityConstraints() throws Exception {
// host_0_1: 8G
// host_1_0: 8G
// host_1_1: 8G
+ // Blacklist:
// node_1_1
// Shouldn't allocate since RR(rack_1) = relax: false
@@ -1838,11 +1841,39 @@ public void testLocalityConstraints() throws Exception {
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0
- // Allow rack-locality for rack_1
+ // Allow rack-locality for rack_1, but blacklist node_1_1
app_0_requests_0.add(
TestUtils.createResourceRequest(rack_1, 1*GB, 1,
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0,
+ Collections.singletonList(host_1_1), null);
+ app_0_requests_0.clear();
+
+ // resourceName:
+ // host_0_0: < 1, 1GB, 1, true >
+ // host_0_1: < null >
+ // rack_0: < null >
+ // host_1_0: < 1, 1GB, 1, true >
+ // host_1_1: < null >
+ // rack_1: < 1, 1GB, 1, true >
+ // ANY: < 1, 1GB, 1, false >
+ // Availability:
+ // host_0_0: 8G
+ // host_0_1: 8G
+ // host_1_0: 8G
+ // host_1_1: 8G
+ // Blacklist: < host_0_0 , host_1_1 > <----
+
+ // node_1_1
+ // Shouldn't allocate since node_1_1 is blacklisted
+ a.assignContainers(clusterResource, node_1_1);
+ verify(app_0, never()).allocate(any(NodeType.class), eq(node_1_1),
+ any(Priority.class), any(ResourceRequest.class), any(Container.class));
+ assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0
+
+ // Now, remove node_1_1 from blacklist, but add rack_1 to blacklist
+ app_0.updateResourceRequests(app_0_requests_0,
+ Collections.singletonList(rack_1), Collections.singletonList(host_1_1));
app_0_requests_0.clear();
// resourceName:
@@ -1851,15 +1882,42 @@ public void testLocalityConstraints() throws Exception {
// rack_0: < null >
// host_1_0: < 1, 1GB, 1, true >
// host_1_1: < null >
- // rack_1: < 1, 1GB, 1, true > <----
+ // rack_1: < 1, 1GB, 1, true >
// ANY: < 1, 1GB, 1, false >
// Availability:
// host_0_0: 8G
// host_0_1: 8G
// host_1_0: 8G
// host_1_1: 8G
+ // Blacklist: < host_0_0 , rack_1 > <----
// node_1_1
+ // Shouldn't allocate since rack_1 is blacklisted
+ a.assignContainers(clusterResource, node_1_1);
+ verify(app_0, never()).allocate(any(NodeType.class), eq(node_1_1),
+ any(Priority.class), any(ResourceRequest.class), any(Container.class));
+ assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0
+
+ // Now remove rack_1 from blacklist
+ app_0.updateResourceRequests(app_0_requests_0,
+ null, Collections.singletonList(rack_1));
+ app_0_requests_0.clear();
+
+ // resourceName:
+ // host_0_0: < 1, 1GB, 1, true >
+ // host_0_1: < null >
+ // rack_0: < null >
+ // host_1_0: < 1, 1GB, 1, true >
+ // host_1_1: < null >
+ // rack_1: < 1, 1GB, 1, true >
+ // ANY: < 1, 1GB, 1, false >
+ // Availability:
+ // host_0_0: 8G
+ // host_0_1: 8G
+ // host_1_0: 8G
+ // host_1_1: 8G
+ // Blacklist: < host_0_0 > <----
+
// Now, should allocate since RR(rack_1) = relax: true
a.assignContainers(clusterResource, node_1_1);
verify(app_0).allocate(eq(NodeType.RACK_LOCAL), eq(node_1_1),
@@ -1874,7 +1932,7 @@ public void testLocalityConstraints() throws Exception {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
false, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateResourceRequests(app_0_requests_0, null, null);
app_0_requests_0.clear();
// resourceName:
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 328daef..bfda5e5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -206,7 +206,7 @@ private ApplicationAttemptId createSchedulingRequest(int memory, int vcores,
ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY,
priority, numContainers, true);
ask.add(request);
- scheduler.allocate(id, ask, new ArrayList());
+ scheduler.allocate(id, ask, new ArrayList(), null, null);
return id;
}
@@ -221,7 +221,7 @@ private void createSchedulingRequestExistingApplication(ResourceRequest request,
ApplicationAttemptId attId) {
List ask = new ArrayList();
ask.add(request);
- scheduler.allocate(attId, ask, new ArrayList());
+ scheduler.allocate(attId, ask, new ArrayList(), null, null);
}
// TESTS
@@ -528,7 +528,7 @@ public void testQueueDemandCalculation() throws Exception {
ResourceRequest request1 =
createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1, true);
ask1.add(request1);
- scheduler.allocate(id11, ask1, new ArrayList());
+ scheduler.allocate(id11, ask1, new ArrayList(), null, null);
// Second ask, queue2 requests 1 large + (2 * minReqSize)
List ask2 = new ArrayList();
@@ -538,14 +538,14 @@ public void testQueueDemandCalculation() throws Exception {
false);
ask2.add(request2);
ask2.add(request3);
- scheduler.allocate(id21, ask2, new ArrayList());
+ scheduler.allocate(id21, ask2, new ArrayList(), null, null);
// Third ask, queue2 requests 1 large
List ask3 = new ArrayList();
ResourceRequest request4 =
createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1, true);
ask3.add(request4);
- scheduler.allocate(id22, ask3, new ArrayList());
+ scheduler.allocate(id22, ask3, new ArrayList(), null, null);
scheduler.update();
@@ -1369,7 +1369,7 @@ public void testReservationWhileMultiplePriorities() {
// Complete container
scheduler.allocate(attId, new ArrayList(),
- Arrays.asList(containerId));
+ Arrays.asList(containerId), null, null);
assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB());
// Schedule at opening
@@ -1444,7 +1444,7 @@ public void testMultipleNodesSingleRackRequest() throws Exception {
asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1, true));
asks.add(createResourceRequest(1024, ResourceRequest.ANY, 1, 2, true));
- scheduler.allocate(appId, asks, new ArrayList());
+ scheduler.allocate(appId, asks, new ArrayList(), null, null);
// node 1 checks in
scheduler.update();
@@ -1799,7 +1799,7 @@ public void testCancelStrictLocality() {
createResourceRequest(1024, node1.getHostName(), 1, 0, true),
createResourceRequest(1024, "rack1", 1, 0, true),
createResourceRequest(1024, ResourceRequest.ANY, 1, 1, true));
- scheduler.allocate(attId1, update, new ArrayList());
+ scheduler.allocate(attId1, update, new ArrayList(), null, null);
// then node2 should get the container
scheduler.handle(node2UpdateEvent);
@@ -1842,7 +1842,7 @@ public void testReservationsStrictLocality() {
anyRequest = createResourceRequest(1024, ResourceRequest.ANY,
1, 1, false);
scheduler.allocate(attId, Arrays.asList(rackRequest, anyRequest),
- new ArrayList());
+ new ArrayList(), null, null);
scheduler.handle(nodeUpdateEvent);
assertEquals(0, app.getReservedContainers().size());
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 4fe3447..5af6585 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -186,7 +186,7 @@ public void testNodeLocalAssignment() throws Exception {
ask.add(nodeLocal);
ask.add(rackLocal);
ask.add(any);
- scheduler.allocate(appAttemptId, ask, new ArrayList());
+ scheduler.allocate(appAttemptId, ask, new ArrayList(), null, null);
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);