diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 48763b3..df35438 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -33,6 +33,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.Priority; @Public @Evolving @@ -465,6 +466,9 @@ /** Default queue name */ public static final String DEFAULT_QUEUE_NAME = "default"; + /** Default application priority */ + public static final Priority DEFAULT_APPLICATION_PRIORITY = Priority.newInstance(10); + /** * Buckets (in minutes) for the number of apps running in each queue. */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java index 401cfec..b693d32 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java @@ -21,6 +21,7 @@ import java.util.Iterator; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import com.google.common.collect.Iterators; @@ -59,6 +60,10 @@ public static String newQueue() { } } + public static Priority newPriority() { + return Priority.newInstance((int) (Math.random() * 1000) + 1); + } + public static ApplicationId newAppID(int i) { return ApplicationId.newInstance(TS, i); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index d8554bd..d519f07 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -515,6 +515,9 @@ public SubmitApplicationResponse submitApplication( if (submissionContext.getQueue() == null) { submissionContext.setQueue(YarnConfiguration.DEFAULT_QUEUE_NAME); } + if (submissionContext.getPriority() == null) { + submissionContext.setPriority(YarnConfiguration.DEFAULT_APPLICATION_PRIORITY); + } if (submissionContext.getApplicationName() == null) { submissionContext.setApplicationName( YarnConfiguration.DEFAULT_APPLICATION_NAME); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 391ccf6..7eb3978 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -87,6 +88,12 @@ * @return the queue to which the application was submitted to. */ String getQueue(); + + /** + * Priority of this application. + * @return the priority {@link Priority} for this application + */ + Priority getApplicationPriority(); /** * Reflects a change in the application's queue from the one specified in the diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index f4f2e20..c425ba5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; @@ -91,6 +92,7 @@ private final Configuration conf; private final String user; private final String name; + private final Priority applicationPriority; private final ApplicationSubmissionContext submissionContext; private final Dispatcher dispatcher; private final YarnScheduler scheduler; @@ -321,6 +323,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, this.startTime = System.currentTimeMillis(); this.applicationType = applicationType; this.applicationTags = applicationTags; + this.applicationPriority = submissionContext.getPriority(); int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -390,6 +393,11 @@ public String getUser() { } @Override + public Priority getApplicationPriority() { + return this.applicationPriority; + } + + @Override public float getProgress() { this.readLock.lock(); @@ -786,7 +794,7 @@ public void transition(RMAppImpl app, RMAppEvent event) { } } app.handler.handle(new AppAddedSchedulerEvent(app.applicationId, - app.submissionContext.getQueue(), app.user)); + app.submissionContext.getQueue(), app.user, app.applicationPriority)); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index de71f71..43ae352 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.util.resource.Resources; @@ -55,6 +56,7 @@ private final String queueName; Queue queue; final String user; + private final Priority applicationPriority; private final AtomicInteger containerIdCounter = new AtomicInteger(0); final Set priorities = new TreeSet( @@ -71,11 +73,19 @@ public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, Queue queue, ActiveUsersManager activeUsersManager) { + this(appAttemptId, user, queue, + YarnConfiguration.DEFAULT_APPLICATION_PRIORITY, activeUsersManager); + } + + public AppSchedulingInfo(ApplicationAttemptId appAttemptId, + String user, Queue queue, Priority applicationPriority, + ActiveUsersManager activeUsersManager) { this.applicationAttemptId = appAttemptId; this.applicationId = appAttemptId.getApplicationId(); this.queue = queue; this.queueName = queue.getQueueName(); this.user = user; + this.applicationPriority = applicationPriority; this.activeUsersManager = activeUsersManager; } @@ -95,6 +105,10 @@ public String getUser() { return user; } + public Priority getApplicationPriority() { + return applicationPriority; + } + public synchronized boolean isPending() { return pending; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java index 2c788aa..78a55e2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java @@ -19,6 +19,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @Private @@ -27,11 +29,18 @@ private Queue queue; private final String user; + private final Priority applicationPriority; private T currentAttempt; public SchedulerApplication(Queue queue, String user) { + this(queue, user, YarnConfiguration.DEFAULT_APPLICATION_PRIORITY); + } + + public SchedulerApplication(Queue queue, String user, + Priority applicationPriority) { this.queue = queue; this.user = user; + this.applicationPriority = applicationPriority; } public Queue getQueue() { @@ -46,6 +55,10 @@ public String getUser() { return user; } + public Priority getApplicationPriority() { + return applicationPriority; + } + public T getCurrentAppAttempt() { return currentAttempt; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index fc7e047..1cbffee 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -140,6 +140,10 @@ public String getUser() { return appSchedulingInfo.getUser(); } + public Priority getApplicationPriority() { + return appSchedulingInfo.getApplicationPriority(); + } + public Map getResourceRequests(Priority priority) { return appSchedulingInfo.getResourceRequests(priority); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java index d6fb36d..debcb7e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java @@ -19,19 +19,29 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.conf.YarnConfiguration; public class AppAddedSchedulerEvent extends SchedulerEvent { private final ApplicationId applicationId; private final String queue; private final String user; + private final Priority applicationPriority; public AppAddedSchedulerEvent( ApplicationId applicationId, String queue, String user) { + this(applicationId, queue, user, + YarnConfiguration.DEFAULT_APPLICATION_PRIORITY); + } + + public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, + String user, Priority applicationPriority) { super(SchedulerEventType.APP_ADDED); this.applicationId = applicationId; this.queue = queue; this.user = user; + this.applicationPriority = applicationPriority; } public ApplicationId getApplicationId() { @@ -46,4 +56,7 @@ public String getUser() { return user; } + public Priority getApplicationPriority() { + return applicationPriority; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index 9ed5179..63236fb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -65,7 +65,7 @@ public AppSchedulable(FairScheduler scheduler, FSSchedulerApp app, FSLeafQueue q this.queue = queue; this.containerTokenSecretManager = scheduler. getContainerTokenSecretManager(); - this.priority = Priority.newInstance(1); + this.priority = app.getApplicationPriority(); this.resourceWeights = new ResourceWeights(); } @@ -139,8 +139,6 @@ public ResourceWeights getWeights() { @Override public Priority getPriority() { - // Right now per-app priorities are not passed to scheduler, - // so everyone has the same priority. return priority; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 830f6f7..91c2242 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -558,12 +558,17 @@ public FairSchedulerEventLog getEventLog() { } /** - * Add a new application to the scheduler, with a given id, queue name, and - * user. This will accept a new app even if the user or queue is above + * Add a new application to the scheduler, with a given id, queue name, user + * and priority. This will accept a new app even if the user or queue is above * configured limits, but the app will not be marked as runnable. */ protected synchronized void addApplication(ApplicationId applicationId, String queueName, String user) { + addApplication(applicationId, queueName, user, + YarnConfiguration.DEFAULT_APPLICATION_PRIORITY); + } + protected synchronized void addApplication(ApplicationId applicationId, + String queueName, String user, Priority applicationPriority) { if (queueName == null || queueName.isEmpty()) { String message = "Reject application " + applicationId + " submitted by user " + user + " with an empty queue name."; @@ -593,13 +598,14 @@ protected synchronized void addApplication(ApplicationId applicationId, } SchedulerApplication application = - new SchedulerApplication(queue, user); + new SchedulerApplication( + queue, user, applicationPriority); applications.put(applicationId, application); queue.getMetrics().submitApp(user); LOG.info("Accepted application " + applicationId + " from user: " + user - + ", in queue: " + queueName + ", currently num of applications: " - + applications.size()); + + ", in queue: " + queueName + ", with priority: " + applicationPriority + + ", currently num of applications: " + applications.size()); rmContext.getDispatcher().getEventHandler() .handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED)); } @@ -1119,8 +1125,8 @@ public void handle(SchedulerEvent event) { throw new RuntimeException("Unexpected event type: " + event); } AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event; - addApplication(appAddedEvent.getApplicationId(), - appAddedEvent.getQueue(), appAddedEvent.getUser()); + addApplication(appAddedEvent.getApplicationId(), appAddedEvent.getQueue(), + appAddedEvent.getUser(), appAddedEvent.getApplicationPriority()); break; case APP_REMOVED: if (!(event instanceof AppRemovedSchedulerEvent)) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index ce5dd96..deb56a8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -71,6 +71,7 @@ private AtomicInteger numAttempts = new AtomicInteger(0); final private String user; final private String queue; + final private Priority applicationPriority; final private ApplicationId applicationId; final private ApplicationAttemptId applicationAttemptId; final private ResourceManager resourceManager; @@ -104,16 +105,23 @@ public Application(String user, ResourceManager resourceManager) public Application(String user, String queue, ResourceManager resourceManager) throws YarnException { + this(user, queue, YarnConfiguration.DEFAULT_APPLICATION_PRIORITY, + resourceManager); + } + + public Application(String user, String queue, Priority applicationPriority, + ResourceManager resourceManager) throws YarnException { this.user = user; this.queue = queue; + this.applicationPriority = applicationPriority; this.resourceManager = resourceManager; // register an application GetNewApplicationRequest request = - Records.newRecord(GetNewApplicationRequest.class); - GetNewApplicationResponse newApp = + Records.newRecord(GetNewApplicationRequest.class); + GetNewApplicationResponse newApp = this.resourceManager.getClientRMService().getNewApplication(request); this.applicationId = newApp.getApplicationId(); - + this.applicationAttemptId = ApplicationAttemptId.newInstance(this.applicationId, this.numAttempts.getAndIncrement()); @@ -127,6 +135,10 @@ public String getQueue() { return queue; } + public Priority getApplicationPriority() { + return applicationPriority; + } + public ApplicationId getApplicationId() { return applicationId; } @@ -151,6 +163,7 @@ public synchronized void submit() throws IOException, YarnException { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(this.applicationId); context.setQueue(this.queue); + context.setPriority(this.applicationPriority); // Set up the container launch context for the application master ContainerLaunchContext amContainer @@ -168,7 +181,8 @@ public synchronized void submit() throws IOException, YarnException { // Notify scheduler AppAddedSchedulerEvent addAppEvent = - new AppAddedSchedulerEvent(this.applicationId, this.queue, "user"); + new AppAddedSchedulerEvent(this.applicationId, this.queue, "user", + this.applicationPriority); scheduler.handle(addAppEvent); AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 77530cf..2b5e19e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -43,6 +43,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; +import org.apache.hadoop.yarn.api.records.Priority; import org.junit.Assert; import org.apache.commons.logging.Log; @@ -551,7 +552,7 @@ public void handle(Event event) {} new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null); - // without name and queue + // without name, queue and priority SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest( appId1, null, null); @@ -566,15 +567,21 @@ public void handle(Event event) {} YarnConfiguration.DEFAULT_APPLICATION_NAME, app1.getName()); Assert.assertEquals("app queue doesn't match", YarnConfiguration.DEFAULT_QUEUE_NAME, app1.getQueue()); + Assert.assertEquals("app priority doesn't match", + YarnConfiguration.DEFAULT_APPLICATION_PRIORITY, + app1.getApplicationPriority()); - // with name and queue + // with name, queue and priority String name = MockApps.newAppName(); String queue = MockApps.newQueue(); + Priority priority = MockApps.newPriority(); ApplicationId appId2 = getApplicationId(101); SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest( appId2, name, queue); submitRequest2.getApplicationSubmissionContext().setApplicationType( "matchType"); + submitRequest2.getApplicationSubmissionContext().setPriority(priority); + try { rmService.submitApplication(submitRequest2); } catch (YarnException e) { @@ -584,6 +591,8 @@ public void handle(Event event) {} Assert.assertNotNull("app doesn't exist", app2); Assert.assertEquals("app name doesn't match", name, app2.getName()); Assert.assertEquals("app queue doesn't match", queue, app2.getQueue()); + Assert.assertEquals("app priority doesn't match", + priority.getPriority(), app2.getApplicationPriority().getPriority()); // duplicate appId try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 2cdbf95..7ffdffe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -67,6 +68,11 @@ public String getQueue() { } @Override + public Priority getApplicationPriority() { + throw new UnsupportedOperationException("Nor supported yet."); + } + + @Override public long getStartTime() { throw new UnsupportedOperationException("Not supported yet."); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index b07525d..8f33b7c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -41,6 +42,7 @@ String user = MockApps.newUserName(); String name = MockApps.newAppName(); String queue = MockApps.newQueue(); + Priority applicationPriority = MockApps.newPriority(); long start = System.currentTimeMillis() - (int) (Math.random() * DT); long submit = start - (int) (Math.random() * DT); long finish = 0; @@ -97,6 +99,15 @@ public void setUser(String user) { } @Override + public Priority getApplicationPriority() { + return applicationPriority; + } + + public void setApplicationPriority(Priority applicationPriority) { + this.applicationPriority = applicationPriority; + } + + @Override public float getProgress() { return (float) 0.0; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java index 5f92676..3aa247d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -134,8 +134,16 @@ protected ApplicationAttemptId createSchedulingRequest( protected ApplicationAttemptId createSchedulingRequest( int memory, int vcores, String queueId, String userId, int numContainers, int priority) { + return createSchedulingRequest(memory, vcores, queueId, userId, + YarnConfiguration.DEFAULT_APPLICATION_PRIORITY, numContainers, priority); + } + + protected ApplicationAttemptId createSchedulingRequest( + int memory, int vcores, String queueId, String userId, + Priority applicationPriority, int numContainers, int priority) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(id.getApplicationId(), queueId, userId); + scheduler.addApplication( + id.getApplicationId(), queueId, userId, applicationPriority); // This conditional is for testAclSubmitApplication where app is rejected // and no app is added. if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) {