From 38c9bff530cc9e72a3c00ab9805c74a8a9a13e72 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Thu, 28 Dec 2017 11:38:50 +0530 Subject: [PATCH] YARN-7666 --- .../api/records/ApplicationSubmissionContext.java | 21 +++++++ .../src/main/proto/yarn_protos.proto | 1 + .../pb/ApplicationSubmissionContextPBImpl.java | 73 +++++++++++++++++++++- .../yarn/server/resourcemanager/rmapp/RMApp.java | 7 +++ .../server/resourcemanager/rmapp/RMAppImpl.java | 15 ++++- .../scheduler/AppSchedulingInfo.java | 41 ++++++++---- .../scheduler/ApplicationPlacementFactory.java | 62 ++++++++++++++++++ .../scheduler/SchedulerApplicationAttempt.java | 13 +++- .../scheduler/placement/AppPlacementAllocator.java | 9 +++ .../placement/LocalityAppPlacementAllocator.java | 5 ++ .../applicationsmanager/MockAsm.java | 5 ++ .../server/resourcemanager/rmapp/MockRMApp.java | 5 ++ .../scheduler/TestAppSchedulingInfo.java | 7 ++- 13 files changed, 242 insertions(+), 22 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ApplicationPlacementFactory.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index 38db60cd9e1..bcc966581f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -598,4 +598,25 @@ public abstract void setLogAggregationContext( @Unstable public abstract void setApplicationTimeouts( Map applicationTimeouts); + + /** + * Get application scheduling environment variables stored as a key value + * pair map for application. + * + * @return placement envs for application. + */ + @Public + @Unstable + public abstract Map getApplicationSchedulingEnvsMap(); + + /** + * Set the scheduling envs for the application. + * + * @param schedulingEnvMap + * A map of env's for the application scheduling preferences. + */ + @Public + @Unstable + public abstract void setApplicationSchedulingEnvsMap( + Map schedulingEnvMap); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 3a9662b2540..acf666eb0a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -459,6 +459,7 @@ message ApplicationSubmissionContextProto { optional string node_label_expression = 16; repeated ResourceRequestProto am_container_resource_request = 17; repeated ApplicationTimeoutMapProto application_timeouts = 18; + repeated StringStringMapProto application_scheduling_envs = 19; } enum ApplicationTimeoutTypeProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index e3dbf4055be..b777ebbb8cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnProtos.StringStringMapProto; import com.google.protobuf.TextFormat; @@ -71,6 +72,7 @@ private LogAggregationContext logAggregationContext = null; private ReservationId reservationId = null; private Map applicationTimeouts = null; + private Map schedulingEnvs = null; public ApplicationSubmissionContextPBImpl() { builder = ApplicationSubmissionContextProto.newBuilder(); @@ -141,6 +143,9 @@ private void mergeLocalToBuilder() { if (this.applicationTimeouts != null) { addApplicationTimeouts(); } + if (this.schedulingEnvs != null) { + addApplicationSchedulingEnvs(); + } } private void mergeLocalToProto() { @@ -662,4 +667,70 @@ public void remove() { }; this.builder.addAllApplicationTimeouts(values); } -} + + private void addApplicationSchedulingEnvs() { + maybeInitBuilder(); + builder.clearApplicationSchedulingEnvs(); + if (this.schedulingEnvs == null) { + return; + } + Iterable values = + new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + private Iterator iterator = schedulingEnvs.keySet() + .iterator(); + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public StringStringMapProto next() { + String key = iterator.next(); + return StringStringMapProto.newBuilder() + .setValue(schedulingEnvs.get(key)).setKey(key).build(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + this.builder.addAllApplicationSchedulingEnvs(values); + } + + private void initApplicationSchedulingEnvs() { + if (this.schedulingEnvs != null) { + return; + } + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + List envs = p.getApplicationSchedulingEnvsList(); + this.schedulingEnvs = new HashMap(envs.size()); + for (StringStringMapProto envProto : envs) { + this.schedulingEnvs.put(envProto.getKey(), envProto.getValue()); + } + } + + @Override + public Map getApplicationSchedulingEnvsMap() { + initApplicationSchedulingEnvs(); + return this.schedulingEnvs; + } + + @Override + public void setApplicationSchedulingEnvsMap( + Map schedulingEnvMap) { + if (schedulingEnvMap == null) { + return; + } + initApplicationSchedulingEnvs(); + this.schedulingEnvs.clear(); + this.schedulingEnvs.putAll(schedulingEnvMap); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 39321ccd2dd..c432a96aa6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -47,6 +47,7 @@ .ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.AppPlacementAllocator; /** * The interface to an Application in the ResourceManager. Take a @@ -311,4 +312,10 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * @return ApplicationPlacementContext */ ApplicationPlacementContext getApplicationPlacementContext(); + + /** + * Get the application scheduling environment variables. + * @return Map of envs related to application scheduling preferences. + */ + Map getApplicationSchedulingEnvs(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 38f666b4c7b..16d4233cd4d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -82,7 +82,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.blacklist.BlacklistManager; import org.apache.hadoop.yarn.server.resourcemanager.blacklist.DisabledBlacklistManager; import org.apache.hadoop.yarn.server.resourcemanager.blacklist.SimpleBlacklistManager; - import org.apache.hadoop.yarn.server.resourcemanager.placement .ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; @@ -151,6 +150,7 @@ private final Map updatedNodes = new HashMap<>(); private final String applicationType; private final Set applicationTags; + private Map applicationSchedulingEnvs = new HashMap<>(); private final long attemptFailuresValidityInterval; private boolean amBlacklistingEnabled = false; @@ -489,6 +489,13 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, this.placementContext = placementContext; + // If applications are not explicitly specifying envs, try to pull from + // AM container environment lists. + applicationSchedulingEnvs + .putAll(submissionContext.getAMContainerSpec().getEnvironment()); + applicationSchedulingEnvs + .putAll(submissionContext.getApplicationSchedulingEnvsMap()); + long localLogAggregationStatusTimeout = conf.getLong(YarnConfiguration.LOG_AGGREGATION_STATUS_TIME_OUT_MS, YarnConfiguration.DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS); @@ -2029,10 +2036,14 @@ public void setApplicationPriority(Priority applicationPriority) { /** * Clear Unused fields to free memory. - * @param app */ private void clearUnusedFields() { this.submissionContext.setAMContainerSpec(null); this.submissionContext.setLogAggregationContext(null); } + + @Override + public Map getApplicationSchedulingEnvs() { + return this.applicationSchedulingEnvs; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index e47f0c1d614..aecd05bcaea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -33,6 +33,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -62,6 +63,14 @@ private static final Log LOG = LogFactory.getLog(AppSchedulingInfo.class); + @InterfaceAudience.Private + public static final String ENV_APPLICATION_PLACEMENT_TYPE_CLASS = + "APPLICATION_PLACEMENT_TYPE_CLASS"; + + @InterfaceAudience.Private + public static final Class + DEFAULT_APPLICATION_PLACEMENT_TYPE_CLASS = LocalityAppPlacementAllocator.class; + private final ApplicationId applicationId; private final ApplicationAttemptId applicationAttemptId; private final AtomicLong containerIdCounter; @@ -90,10 +99,12 @@ private final ReentrantReadWriteLock.WriteLock writeLock; public final ContainerUpdateContext updateContext; + public final Map applicationSchedulingEnvs; - public AppSchedulingInfo(ApplicationAttemptId appAttemptId, - String user, Queue queue, AbstractUsersManager abstractUsersManager, - long epoch, ResourceUsage appResourceUsage) { + public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, + Queue queue, AbstractUsersManager abstractUsersManager, long epoch, + ResourceUsage appResourceUsage, + Map applicationSchedulingEnvs) { this.applicationAttemptId = appAttemptId; this.applicationId = appAttemptId.getApplicationId(); this.queue = queue; @@ -102,6 +113,7 @@ public AppSchedulingInfo(ApplicationAttemptId appAttemptId, this.containerIdCounter = new AtomicLong( epoch << ResourceManager.EPOCH_BIT_SHIFT); this.appResourceUsage = appResourceUsage; + this.applicationSchedulingEnvs = applicationSchedulingEnvs; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); updateContext = new ContainerUpdateContext(this); @@ -211,24 +223,27 @@ boolean addRequestToAppPlacement( Map> dedupRequests) { boolean offswitchResourcesUpdated = false; for (Map.Entry> entry : - dedupRequests.entrySet()) { + dedupRequests.entrySet()) { SchedulerRequestKey schedulerRequestKey = entry.getKey(); - if (!schedulerKeyToAppPlacementAllocator.containsKey( - schedulerRequestKey)) { + if (!schedulerKeyToAppPlacementAllocator + .containsKey(schedulerRequestKey)) { + AppPlacementAllocator placementAllocatorInstance = ApplicationPlacementFactory + .getAppPlacementAllocator(applicationSchedulingEnvs + .get(ENV_APPLICATION_PLACEMENT_TYPE_CLASS)); + placementAllocatorInstance.setAppSchedulingInfo(this); + schedulerKeyToAppPlacementAllocator.put(schedulerRequestKey, - new LocalityAppPlacementAllocator<>(this)); + placementAllocatorInstance); } // Update AppPlacementAllocator - PendingAskUpdateResult pendingAmountChanges = - schedulerKeyToAppPlacementAllocator.get(schedulerRequestKey) - .updatePendingAsk(entry.getValue().values(), - recoverPreemptedRequestForAContainer); + PendingAskUpdateResult pendingAmountChanges = schedulerKeyToAppPlacementAllocator + .get(schedulerRequestKey).updatePendingAsk(entry.getValue().values(), + recoverPreemptedRequestForAContainer); if (null != pendingAmountChanges) { - updatePendingResources( - pendingAmountChanges, schedulerRequestKey, + updatePendingResources(pendingAmountChanges, schedulerRequestKey, queue.getMetrics()); offswitchResourcesUpdated = true; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ApplicationPlacementFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ApplicationPlacementFactory.java new file mode 100644 index 00000000000..57b39cb25c0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ApplicationPlacementFactory.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.AppPlacementAllocator; + +/** + * Factory class to build various application placement policies. + */ +@Public +@Unstable +public class ApplicationPlacementFactory { + + /** + * Get AppPlacementAllocator related to the placement type requested. + * + * @param appPlacementAllocatorName + * allocator class name. + * @return Specific AppPlacementAllocator instance based on type + */ + public static AppPlacementAllocator getAppPlacementAllocator( + String appPlacementAllocatorName) { + Class policyClass; + try { + if (appPlacementAllocatorName == null) { + policyClass = AppSchedulingInfo.DEFAULT_APPLICATION_PLACEMENT_TYPE_CLASS; + } else { + policyClass = Class.forName(appPlacementAllocatorName); + } + } catch (ClassNotFoundException e) { + policyClass = AppSchedulingInfo.DEFAULT_APPLICATION_PLACEMENT_TYPE_CLASS; + } + + if (!AppPlacementAllocator.class.isAssignableFrom(policyClass)) { + policyClass = AppSchedulingInfo.DEFAULT_APPLICATION_PLACEMENT_TYPE_CLASS; + } + + @SuppressWarnings("unchecked") + AppPlacementAllocator placementAllocatorInstance = (AppPlacementAllocator) ReflectionUtils + .newInstance(policyClass, null); + return placementAllocatorInstance; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index dfb0e67fcd9..f02f113e9dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -197,6 +197,8 @@ protected ReentrantReadWriteLock.ReadLock readLock; protected ReentrantReadWriteLock.WriteLock writeLock; + private Map applicationSchedulingEnvs = new HashMap<>(); + // Not confirmed allocation resource, will be used to avoid too many proposal // rejected because of duplicated allocation private AtomicLong unconfirmedAllocatedMem = new AtomicLong(); @@ -207,9 +209,6 @@ public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, RMContext rmContext) { Preconditions.checkNotNull(rmContext, "RMContext should not be null"); this.rmContext = rmContext; - this.appSchedulingInfo = - new AppSchedulingInfo(applicationAttemptId, user, queue, - abstractUsersManager, rmContext.getEpoch(), attemptResourceUsage); this.queue = queue; this.pendingRelease = Collections.newSetFromMap( new ConcurrentHashMap()); @@ -227,8 +226,12 @@ public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, this.logAggregationContext = appSubmissionContext.getLogAggregationContext(); } + applicationSchedulingEnvs = rmApp.getApplicationSchedulingEnvs(); } + this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, + queue, abstractUsersManager, rmContext.getEpoch(), attemptResourceUsage, + applicationSchedulingEnvs); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); @@ -1434,4 +1437,8 @@ public String getDiagnosticMessage() { return diagnosticMessage; } } + + public Map getApplicationSchedulingEnvs() { + return this.applicationSchedulingEnvs; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java index dcb38aa0054..5c49450d7ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode; @@ -157,4 +158,12 @@ boolean acceptNodePartition(String nodePartition, * Print human-readable requests to LOG debug. */ void showRequests(); + + /** + * Set app scheduling info. + * + * @param appSchedulingInfo + * app info object. + */ + void setAppSchedulingInfo(AppSchedulingInfo appSchedulingInfo); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java index 766827ca300..db794037fe1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java @@ -419,4 +419,9 @@ public ContainerRequest allocate(SchedulerRequestKey schedulerKey, writeLock.unlock(); } } + + @Override + public void setAppSchedulingInfo(AppSchedulingInfo appSchedulingInfo) { + this.appSchedulingInfo = appSchedulingInfo; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 2aca3752013..72de27cf95a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -254,6 +254,11 @@ public ApplicationPlacementContext getApplicationPlacementContext() { public CollectorInfo getCollectorInfo() { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public Map getApplicationSchedulingEnvs() { + throw new UnsupportedOperationException("Not supported yet."); + } } public static RMApp newApplication(int i) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 756759957e6..c399368593a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -338,4 +338,9 @@ public ApplicationPlacementContext getApplicationPlacementContext() { public CollectorInfo getCollectorInfo() { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public Map getApplicationSchedulingEnvs() { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java index bb29889a854..443a990fd7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java @@ -47,8 +47,9 @@ public void testBacklistChanged() { FSLeafQueue queue = mock(FSLeafQueue.class); doReturn("test").when(queue).getQueueName(); - AppSchedulingInfo appSchedulingInfo = new AppSchedulingInfo( - appAttemptId, "test", queue, null, 0, new ResourceUsage()); + AppSchedulingInfo appSchedulingInfo = new AppSchedulingInfo(appAttemptId, + "test", queue, null, 0, new ResourceUsage(), + null); appSchedulingInfo.updatePlacesBlacklistedByApp(new ArrayList(), new ArrayList()); @@ -120,7 +121,7 @@ public void testSchedulerKeyAccounting() { doReturn(mock(QueueMetrics.class)).when(queue).getMetrics(); AppSchedulingInfo info = new AppSchedulingInfo( appAttemptId, "test", queue, mock(ActiveUsersManager.class), 0, - new ResourceUsage()); + new ResourceUsage(), null); Assert.assertEquals(0, info.getSchedulerKeys().size()); Priority pri1 = Priority.newInstance(1); -- 2.14.3 (Apple Git-98)