From 18f6bfb1719db7152dc84499ed9d9a13bf09a6c8 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 4 Mar 2019 16:45:20 +0530 Subject: [PATCH] YARN-9328 --- .../amrmproxy/AMRMProxyTokenSecretManager.java | 24 ++-- .../application/ApplicationImpl.java | 8 +- .../containermanager/container/ContainerImpl.java | 7 +- .../localizer/LocalizedResource.java | 7 +- .../tracker/NMLogAggregationStatusTracker.java | 2 +- .../resourcemanager/recovery/RMStateStore.java | 8 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 8 +- .../rmapp/attempt/RMAppAttemptImpl.java | 7 +- .../scheduler/capacity/CapacityScheduler.java | 79 ++++++-------- .../scheduler/capacity/LeafQueue.java | 121 ++++++++++----------- .../scheduler/capacity/ManagedParentQueue.java | 8 +- .../scheduler/capacity/ParentQueue.java | 64 +++++------ .../scheduler/capacity/UsersManager.java | 42 ++++--- .../scheduler/fair/FSParentQueue.java | 8 +- .../scheduler/fair/FairScheduler.java | 6 +- .../security/AMRMTokenSecretManager.java | 26 ++--- .../security/NMTokenSecretManagerInRM.java | 2 +- .../security/RMContainerTokenSecretManager.java | 2 +- .../volume/csi/lifecycle/VolumeImpl.java | 20 ++-- 19 files changed, 208 insertions(+), 241 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyTokenSecretManager.java index f36d4da..0fd5c00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyTokenSecretManager.java @@ -128,10 +128,10 @@ public void setNMStateStoreService(NMStateStoreService nmStateStoreService) { } public void applicationMasterFinished(ApplicationAttemptId appAttemptId) { + LOG.info("Application finished, removing password for " + + appAttemptId); this.writeLock.lock(); try { - LOG.info("Application finished, removing password for " - + appAttemptId); this.appAttemptSet.remove(appAttemptId); } finally { this.writeLock.unlock(); @@ -148,9 +148,9 @@ public void run() { @Private @VisibleForTesting public void rollMasterKey() { + LOG.info("Rolling master-key for amrm-tokens"); this.writeLock.lock(); try { - LOG.info("Rolling master-key for amrm-tokens"); this.nextMasterKey = createNewMasterKey(); if (this.nmStateStore != null) { try { @@ -210,9 +210,9 @@ public MasterKeyData createNewMasterKey() { public Token createAndGetAMRMToken( ApplicationAttemptId appAttemptId) { + LOG.info("Create AMRMToken for ApplicationAttempt: " + appAttemptId); this.writeLock.lock(); try { - LOG.info("Create AMRMToken for ApplicationAttempt: " + appAttemptId); AMRMTokenIdentifier identifier = new AMRMTokenIdentifier(appAttemptId, getMasterKey() .getMasterKey().getKeyId()); @@ -244,14 +244,12 @@ public MasterKeyData getMasterKey() { @Override public byte[] retrievePassword(AMRMTokenIdentifier identifier) throws InvalidToken { + ApplicationAttemptId applicationAttemptId = + identifier.getApplicationAttemptId(); + LOG.debug("Trying to retrieve password for {}", applicationAttemptId); + this.readLock.lock(); try { - ApplicationAttemptId applicationAttemptId = - identifier.getApplicationAttemptId(); - if (LOG.isDebugEnabled()) { - LOG.debug("Trying to retrieve password for " - + applicationAttemptId); - } if (!appAttemptSet.contains(applicationAttemptId)) { throw new InvalidToken(applicationAttemptId + " not found in AMRMProxyTokenSecretManager."); @@ -307,11 +305,11 @@ public MasterKeyData getNextMasterKeyData() { @Override @Private protected byte[] createPassword(AMRMTokenIdentifier identifier) { + ApplicationAttemptId applicationAttemptId = + identifier.getApplicationAttemptId(); + LOG.info("Creating password for " + applicationAttemptId); this.readLock.lock(); try { - ApplicationAttemptId applicationAttemptId = - identifier.getApplicationAttemptId(); - LOG.info("Creating password for " + applicationAttemptId); return createPassword(identifier.getBytes(), getMasterKey() .getSecretKey()); } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index ad995fb..baa117c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -635,14 +635,10 @@ public void transition(ApplicationImpl app, ApplicationEvent event) { @Override public void handle(ApplicationEvent event) { + ApplicationId applicationID = event.getApplicationID(); + LOG.debug("Processing {} of type {}", applicationID, event.getType()); this.writeLock.lock(); - try { - ApplicationId applicationID = event.getApplicationID(); - if (LOG.isDebugEnabled()) { - LOG.debug( - "Processing " + applicationID + " of type " + event.getType()); - } ApplicationState oldState = stateMachine.getCurrentState(); ApplicationState newState = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 8aa8d07..3143cea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -2107,13 +2107,10 @@ public void transition(ContainerImpl container, ContainerEvent event) { @Override public void handle(ContainerEvent event) { + ContainerId containerID = event.getContainerID(); + LOG.debug("Processing {} of type {}", containerID, event.getType()); try { this.writeLock.lock(); - - ContainerId containerID = event.getContainerID(); - if (LOG.isDebugEnabled()) { - LOG.debug("Processing " + containerID + " of type " + event.getType()); - } ContainerState oldState = stateMachine.getCurrentState(); ContainerState newState = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java index 7cca7cf..c2ea67d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java @@ -187,13 +187,10 @@ public void unlock() { @Override public void handle(ResourceEvent event) { + Path resourcePath = event.getLocalResourceRequest().getPath(); + LOG.debug("Processing {} of type {}", resourcePath, event.getType()); try { this.writeLock.lock(); - - Path resourcePath = event.getLocalResourceRequest().getPath(); - if (LOG.isDebugEnabled()) { - LOG.debug("Processing " + resourcePath + " of type " + event.getType()); - } ResourceState oldState = this.stateMachine.getCurrentState(); ResourceState newState = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/logaggregation/tracker/NMLogAggregationStatusTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/logaggregation/tracker/NMLogAggregationStatusTracker.java index eb2aaf5..6517d04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/logaggregation/tracker/NMLogAggregationStatusTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/logaggregation/tracker/NMLogAggregationStatusTracker.java @@ -200,10 +200,10 @@ private void rollLogAggregationStatus() { // log aggregation status, we should block the rollLogAggregationStatus // calls as well as pullCachedLogAggregationReports call. So, the // writeLocker is used here. + LOG.info("Rolling over the cached log aggregation status."); this.writeLocker.lock(); try { long currentTimeStamp = System.currentTimeMillis(); - LOG.info("Rolling over the cached log aggregation status."); Iterator> it = recoveryStatuses.entrySet().iterator(); while (it.hasNext()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index deb79a5..230019e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -1231,13 +1231,11 @@ protected boolean isFencedState() { // Dispatcher related code protected void handleStoreEvent(RMStateStoreEvent event) { + if (LOG.isDebugEnabled()) { + LOG.debug("Processing event of type " + event.getType()); + } this.writeLock.lock(); try { - - if (LOG.isDebugEnabled()) { - LOG.debug("Processing event of type " + event.getType()); - } - final RMStateStoreState oldState = getRMStateStoreState(); this.stateMachine.doTransition(event.getType(), event); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 696b39d..97adc32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -903,12 +903,12 @@ public int getMaxAppAttempts() { @Override public void handle(RMAppEvent event) { - this.writeLock.lock(); + ApplicationId appID = event.getApplicationId(); + LOG.debug("Processing event for " + appID + " of type " + + event.getType()); + this.writeLock.lock(); try { - ApplicationId appID = event.getApplicationId(); - LOG.debug("Processing event for " + appID + " of type " - + event.getType()); final RMAppState oldState = getState(); try { /* keep the master in sync with the state machine */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 7b04ae7..e3e3a90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -902,13 +902,12 @@ public void setMasterContainer(Container container) { @Override public void handle(RMAppAttemptEvent event) { + ApplicationAttemptId appAttemptID = event.getApplicationAttemptId(); + LOG.debug("Processing event for " + appAttemptID + " of type " + + event.getType()); this.writeLock.lock(); - try { - ApplicationAttemptId appAttemptID = event.getApplicationAttemptId(); - LOG.debug("Processing event for " + appAttemptID + " of type " - + event.getType()); final RMAppAttemptState oldState = getAppAttemptState(); try { /* keep the master in sync with the state machine */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 8ecc2f0..ece72c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -402,17 +402,16 @@ void initScheduler(Configuration configuration) throws multiNodePlacementEnabled, this.conf.getMultiNodePlacementPolicies()); } - - LOG.info("Initialized CapacityScheduler with " + "calculator=" - + getResourceCalculator().getClass() + ", " + "minimumAllocation=<" - + getMinimumResourceCapability() + ">, " + "maximumAllocation=<" - + getMaximumResourceCapability() + ">, " + "asynchronousScheduling=" - + scheduleAsynchronously + ", " + "asyncScheduleInterval=" - + asyncScheduleInterval + "ms" + ",multiNodePlacementEnabled=" - + multiNodePlacementEnabled); } finally { writeLock.unlock(); } + LOG.info("Initialized CapacityScheduler with " + "calculator=" + + getResourceCalculator().getClass() + ", " + "minimumAllocation=<" + + getMinimumResourceCapability() + ">, " + "maximumAllocation=<" + + getMaximumResourceCapability() + ">, " + "asynchronousScheduling=" + + scheduleAsynchronously + ", " + "asyncScheduleInterval=" + + asyncScheduleInterval + "ms" + ",multiNodePlacementEnabled=" + + multiNodePlacementEnabled); } private void startSchedulerThreads() { @@ -475,13 +474,13 @@ public void serviceStop() throws Exception { public void reinitialize(Configuration newConf, RMContext rmContext) throws IOException { try { + LOG.info("Re-initializing queues..."); writeLock.lock(); Configuration configuration = new Configuration(newConf); CapacitySchedulerConfiguration oldConf = this.conf; this.conf = csConfProvider.loadConfiguration(configuration); validateConf(this.conf); try { - LOG.info("Re-initializing queues..."); refreshMaximumAllocation( ResourceUtils.fetchMaximumAllocationFromConfig(this.conf)); reinitializeQueues(this.conf); @@ -856,15 +855,15 @@ private void addApplicationOnRecovery(ApplicationId applicationId, SchedulerApplication application = new SchedulerApplication(queue, user, priority); applications.put(applicationId, application); - LOG.info("Accepted application " + applicationId + " from user: " + user - + ", in queue: " + queueName); - if (LOG.isDebugEnabled()) { - LOG.debug( - applicationId + " is recovering. Skip notifying APP_ACCEPTED"); - } } finally { writeLock.unlock(); } + LOG.info("Accepted application " + applicationId + " from user: " + user + + ", in queue: " + queueName); + if (LOG.isDebugEnabled()) { + LOG.debug( + applicationId + " is recovering. Skip notifying APP_ACCEPTED"); + } } private CSQueue getOrCreateQueueFromPlacementContext(ApplicationId @@ -1001,13 +1000,13 @@ private void addApplication(ApplicationId applicationId, String queueName, SchedulerApplication application = new SchedulerApplication(queue, user, priority); applications.put(applicationId, application); - LOG.info("Accepted application " + applicationId + " from user: " + user - + ", in queue: " + queueName); rmContext.getDispatcher().getEventHandler().handle( new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED)); } finally { writeLock.unlock(); } + LOG.info("Accepted application " + applicationId + " from user: " + user + + ", in queue: " + queueName); } private void addApplicationAttempt( @@ -1094,11 +1093,10 @@ private void doneApplication(ApplicationId applicationId, private void doneApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) { + LOG.info("Application Attempt " + applicationAttemptId + " is done." + + " finalState=" + rmAppAttemptFinalState); try { writeLock.lock(); - LOG.info("Application Attempt " + applicationAttemptId + " is done." - + " finalState=" + rmAppAttemptFinalState); - FiCaSchedulerApp attempt = getApplicationAttempt(applicationAttemptId); SchedulerApplication application = applications.get( applicationAttemptId.getApplicationId()); @@ -2056,13 +2054,11 @@ private void removeNode(RMNode nodeInfo) { t.suspendSchedule(); } } - - LOG.info( - "Removed node " + nodeInfo.getNodeAddress() + " clusterResource: " - + getClusterResource()); } finally { writeLock.unlock(); } + LOG.info("Removed node " + nodeInfo.getNodeAddress() + " clusterResource: " + + getClusterResource()); } @Override @@ -2159,13 +2155,12 @@ public void killContainer(RMContainer container) { public void markContainerForKillable( RMContainer killableContainer) { + if (LOG.isDebugEnabled()) { + LOG.debug(SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE + ": container" + + killableContainer.toString()); + } try { writeLock.lock(); - if (LOG.isDebugEnabled()) { - LOG.debug(SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE + ": container" - + killableContainer.toString()); - } - if (!isLazyPreemptionEnabled) { super.completedContainer(killableContainer, SchedulerUtils .createPreemptedContainerStatus(killableContainer.getContainerId(), @@ -2195,13 +2190,13 @@ public void markContainerForKillable( private void markContainerForNonKillable( RMContainer nonKillableContainer) { + if (LOG.isDebugEnabled()) { + LOG.debug( + SchedulerEventType.MARK_CONTAINER_FOR_NONKILLABLE + ": container" + + nonKillableContainer.toString()); + } try { writeLock.lock(); - if (LOG.isDebugEnabled()) { - LOG.debug( - SchedulerEventType.MARK_CONTAINER_FOR_NONKILLABLE + ": container" - + nonKillableContainer.toString()); - } FiCaSchedulerNode node = getSchedulerNode( nonKillableContainer.getAllocatedNode()); @@ -2315,9 +2310,9 @@ private String resolveReservationQueueName(String queueName, @Override public void removeQueue(String queueName) throws SchedulerDynamicEditException { + LOG.info("Removing queue: " + queueName); try { writeLock.lock(); - LOG.info("Removing queue: " + queueName); CSQueue q = this.getQueue(queueName); if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom( q.getClass()))) { @@ -2339,11 +2334,11 @@ public void removeQueue(String queueName) ((AbstractManagedParentQueue) disposableLeafQueue.getParent()) .removeChildQueue(q); this.queueManager.removeQueue(queueName); - LOG.info( - "Removal of AutoCreatedLeafQueue " + queueName + " has succeeded"); } finally { writeLock.unlock(); } + LOG.info( + "Removal of AutoCreatedLeafQueue " + queueName + " has succeeded"); } @Override @@ -3060,14 +3055,12 @@ public CapacitySchedulerQueueManager getCapacitySchedulerQueueManager() { */ public boolean moveReservedContainer(RMContainer toBeMovedContainer, FiCaSchedulerNode targetNode) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to move container=" + toBeMovedContainer + " to node=" + + targetNode.getNodeID()); + } try { writeLock.lock(); - - if (LOG.isDebugEnabled()) { - LOG.debug("Trying to move container=" + toBeMovedContainer + " to node=" - + targetNode.getNodeID()); - } - FiCaSchedulerNode sourceNode = getNode(toBeMovedContainer.getNodeId()); if (null == sourceNode) { if (LOG.isDebugEnabled()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 3920987..fa5e189 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -168,6 +168,8 @@ protected void setupQueueConfigs(Resource clusterResource) protected void setupQueueConfigs(Resource clusterResource, CapacitySchedulerConfiguration conf) throws IOException { + StringBuilder aclsString = new StringBuilder(); + StringBuilder labelStrBuilder = new StringBuilder(); try { writeLock.lock(); CapacitySchedulerConfiguration schedConf = csContext.getConfiguration(); @@ -241,12 +243,10 @@ protected void setupQueueConfigs(Resource clusterResource, Resources.subtract(maximumAllocation, minimumAllocation), maximumAllocation); - StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : acls.entrySet()) { aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); } - StringBuilder labelStrBuilder = new StringBuilder(); if (accessibleLabels != null) { for (String s : accessibleLabels) { labelStrBuilder.append(s) @@ -280,61 +280,59 @@ protected void setupQueueConfigs(Resource clusterResource, + getQueuePath() + ")"); } } - usersManager.updateUserWeights(); - - LOG.info( - "Initializing " + queueName + "\n" + "capacity = " + queueCapacities - .getCapacity() + " [= (float) configuredCapacity / 100 ]" + "\n" - + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() - + " [= parentAbsoluteCapacity * capacity ]" + "\n" - + "maxCapacity = " + queueCapacities.getMaximumCapacity() - + " [= configuredMaxCapacity ]" + "\n" + "absoluteMaxCapacity = " - + queueCapacities.getAbsoluteMaximumCapacity() - + " [= 1.0 maximumCapacity undefined, " - + "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" - + "\n" + "effectiveMinResource=" + - getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) + "\n" - + " , effectiveMaxResource=" + - getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL) - + "\n" + "userLimit = " + usersManager.getUserLimit() - + " [= configuredUserLimit ]" + "\n" + "userLimitFactor = " - + usersManager.getUserLimitFactor() - + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = " - + maxApplications - + " [= configuredMaximumSystemApplicationsPerQueue or" - + " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" - + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser - + " [= (int)(maxApplications * (userLimit / 100.0f) * " - + "userLimitFactor) ]" + "\n" + "usedCapacity = " - + queueCapacities.getUsedCapacity() + " [= usedResourcesMemory / " - + "(clusterResourceMemory * absoluteCapacity)]" + "\n" - + "absoluteUsedCapacity = " + absoluteUsedCapacity - + " [= usedResourcesMemory / clusterResourceMemory]" + "\n" - + "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent - + " [= configuredMaximumAMResourcePercent ]" + "\n" - + "minimumAllocationFactor = " + minimumAllocationFactor - + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " - + "maximumAllocationMemory ]" + "\n" + "maximumAllocation = " - + maximumAllocation + " [= configuredMaxAllocation ]" + "\n" - + "numContainers = " + numContainers - + " [= currentNumContainers ]" + "\n" + "state = " + getState() - + " [= configuredState ]" + "\n" + "acls = " + aclsString - + " [= configuredAcls ]" + "\n" - + "nodeLocalityDelay = " + nodeLocalityDelay + "\n" - + "rackLocalityAdditionalDelay = " - + rackLocalityAdditionalDelay + "\n" - + "labels=" + labelStrBuilder.toString() + "\n" - + "reservationsContinueLooking = " - + reservationsContinueLooking + "\n" + "preemptionDisabled = " - + getPreemptionDisabled() + "\n" + "defaultAppPriorityPerQueue = " - + defaultAppPriorityPerQueue + "\npriority = " + priority - + "\nmaxLifetime = " + maxApplicationLifetime + " seconds" - + "\ndefaultLifetime = " - + defaultApplicationLifetime + " seconds"); } finally { writeLock.unlock(); } + LOG.info( + "Initializing " + queueName + "\n" + "capacity = " + queueCapacities + .getCapacity() + " [= (float) configuredCapacity / 100 ]" + "\n" + + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() + + " [= parentAbsoluteCapacity * capacity ]" + "\n" + + "maxCapacity = " + queueCapacities.getMaximumCapacity() + + " [= configuredMaxCapacity ]" + "\n" + "absoluteMaxCapacity = " + + queueCapacities.getAbsoluteMaximumCapacity() + + " [= 1.0 maximumCapacity undefined, " + + "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" + + "\n" + "effectiveMinResource=" + + getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) + "\n" + + " , effectiveMaxResource=" + + getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL) + + "\n" + "userLimit = " + usersManager.getUserLimit() + + " [= configuredUserLimit ]" + "\n" + "userLimitFactor = " + + usersManager.getUserLimitFactor() + + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = " + + maxApplications + + " [= configuredMaximumSystemApplicationsPerQueue or" + + " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" + + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser + + " [= (int)(maxApplications * (userLimit / 100.0f) * " + + "userLimitFactor) ]" + "\n" + "usedCapacity = " + + queueCapacities.getUsedCapacity() + " [= usedResourcesMemory / " + + "(clusterResourceMemory * absoluteCapacity)]" + "\n" + + "absoluteUsedCapacity = " + absoluteUsedCapacity + + " [= usedResourcesMemory / clusterResourceMemory]" + "\n" + + "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent + + " [= configuredMaximumAMResourcePercent ]" + "\n" + + "minimumAllocationFactor = " + minimumAllocationFactor + + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " + + "maximumAllocationMemory ]" + "\n" + "maximumAllocation = " + + maximumAllocation + " [= configuredMaxAllocation ]" + "\n" + + "numContainers = " + numContainers + + " [= currentNumContainers ]" + "\n" + "state = " + getState() + + " [= configuredState ]" + "\n" + "acls = " + aclsString + + " [= configuredAcls ]" + "\n" + + "nodeLocalityDelay = " + nodeLocalityDelay + "\n" + + "rackLocalityAdditionalDelay = " + + rackLocalityAdditionalDelay + "\n" + + "labels=" + labelStrBuilder.toString() + "\n" + + "reservationsContinueLooking = " + + reservationsContinueLooking + "\n" + "preemptionDisabled = " + + getPreemptionDisabled() + "\n" + "defaultAppPriorityPerQueue = " + + defaultAppPriorityPerQueue + "\npriority = " + priority + + "\nmaxLifetime = " + maxApplicationLifetime + " seconds" + + "\ndefaultLifetime = " + + defaultApplicationLifetime + " seconds"); } /** @@ -934,18 +932,17 @@ private void addApplicationAttempt(FiCaSchedulerApp application, + application.getApplicationAttemptId() + " since cluster resource is " + Resources.none()); } - - LOG.info( - "Application added -" + " appId: " + application.getApplicationId() - + " user: " + application.getUser() + "," + " leaf-queue: " - + getQueueName() + " #user-pending-applications: " + user - .getPendingApplications() + " #user-active-applications: " + user - .getActiveApplications() + " #queue-pending-applications: " - + getNumPendingApplications() + " #queue-active-applications: " - + getNumActiveApplications()); } finally { writeLock.unlock(); } + LOG.info( + "Application added -" + " appId: " + application.getApplicationId() + + " user: " + application.getUser() + "," + " leaf-queue: " + + getQueueName() + " #user-pending-applications: " + user + .getPendingApplications() + " #user-active-applications: " + user + .getActiveApplications() + " #queue-pending-applications: " + + getNumPendingApplications() + " #queue-active-applications: " + + getNumActiveApplications()); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 6788bb4..61f4586 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -115,10 +115,6 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) validateAndApplyQueueManagementChanges(queueManagementChanges); - LOG.info( - "Reinitialized Managed Parent Queue: [{}] with capacity [{}]" - + " with max capacity [{}]", - queueName, super.getCapacity(), super.getMaximumCapacity()); } catch (YarnException ye) { LOG.error("Exception while computing policy changes for leaf queue : " + getQueueName(), ye); @@ -126,6 +122,10 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) } finally { writeLock.unlock(); } + LOG.info( + "Reinitialized Managed Parent Queue: [{}] with capacity [{}]" + + " with max capacity [{}]", + queueName, super.getCapacity(), super.getMaximumCapacity()); } private void initializeQueueManagementPolicy() throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 6d3794e..1876fb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -125,15 +125,15 @@ private String getQueueOrderingPolicyConfigName() { protected void setupQueueConfigs(Resource clusterResource) throws IOException { + StringBuilder aclsString = new StringBuilder(); + StringBuilder labelStrBuilder = new StringBuilder(); try { writeLock.lock(); super.setupQueueConfigs(clusterResource); - StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : acls.entrySet()) { aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); } - StringBuilder labelStrBuilder = new StringBuilder(); if (accessibleLabels != null) { for (String s : accessibleLabels) { labelStrBuilder.append(s) @@ -147,19 +147,18 @@ protected void setupQueueConfigs(Resource clusterResource) null : ((ParentQueue) parent).getQueueOrderingPolicyConfigName()); queueOrderingPolicy.setQueues(childQueues); - - LOG.info(queueName + ", capacity=" + this.queueCapacities.getCapacity() - + ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() - + ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() - + ", absoluteMaxCapacity=" + this.queueCapacities - .getAbsoluteMaximumCapacity() + ", state=" + getState() + ", acls=" - + aclsString + ", labels=" + labelStrBuilder.toString() + "\n" - + ", reservationsContinueLooking=" + reservationsContinueLooking - + ", orderingPolicy=" + getQueueOrderingPolicyConfigName() - + ", priority=" + priority); } finally { writeLock.unlock(); } + LOG.info(queueName + ", capacity=" + this.queueCapacities.getCapacity() + + ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() + + ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() + + ", absoluteMaxCapacity=" + this.queueCapacities + .getAbsoluteMaximumCapacity() + ", state=" + getState() + ", acls=" + + aclsString + ", labels=" + labelStrBuilder.toString() + "\n" + + ", reservationsContinueLooking=" + reservationsContinueLooking + + ", orderingPolicy=" + getQueueOrderingPolicyConfigName() + + ", priority=" + priority); } private static float PRECISION = 0.0005f; // 0.05% precision @@ -245,12 +244,12 @@ void setChildQueues(Collection childQueues) { this.childQueues.clear(); this.childQueues.addAll(childQueues); - if (LOG.isDebugEnabled()) { - LOG.debug("setChildQueues: " + getChildQueuesToPrint()); - } } finally { writeLock.unlock(); } + if (LOG.isDebugEnabled()) { + LOG.debug("setChildQueues: " + getChildQueuesToPrint()); + } } @Override @@ -490,14 +489,13 @@ private void addApplication(ApplicationId applicationId, try { writeLock.lock(); ++numApplications; - - LOG.info( - "Application added -" + " appId: " + applicationId + " user: " + user - + " leaf-queue of parent: " + getQueueName() + " #applications: " - + getNumApplications()); } finally { writeLock.unlock(); } + LOG.info( + "Application added -" + " appId: " + applicationId + " user: " + user + + " leaf-queue of parent: " + getQueueName() + " #applications: " + + getNumApplications()); } @Override @@ -518,13 +516,12 @@ private void removeApplication(ApplicationId applicationId, try { writeLock.lock(); --numApplications; - - LOG.info("Application removed -" + " appId: " + applicationId + " user: " - + user + " leaf-queue of parent: " + getQueueName() - + " #applications: " + getNumApplications()); } finally { writeLock.unlock(); } + LOG.info("Application removed -" + " appId: " + applicationId + " user: " + + user + " leaf-queue of parent: " + getQueueName() + + " #applications: " + getNumApplications()); } private String getParentName() { @@ -857,15 +854,13 @@ private void internalReleaseResource(Resource clusterResource, writeLock.lock(); super.releaseResource(clusterResource, releasedResource, node.getPartition()); - - if (LOG.isDebugEnabled()) { - LOG.debug( - "completedContainer " + this + ", cluster=" + clusterResource); - } - } finally { writeLock.unlock(); } + if (LOG.isDebugEnabled()) { + LOG.debug( + "completedContainer " + this + ", cluster=" + clusterResource); + } } @Override @@ -1336,14 +1331,13 @@ public void apply(Resource cluster, // Note: Update headroom to account for current allocation too... allocateResource(cluster, allocation.getAllocatedOrReservedResource(), schedulerContainer.getNodePartition()); - - LOG.info("assignedContainer" + " queue=" + getQueueName() - + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" - + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() - + " cluster=" + cluster); } finally { writeLock.unlock(); } + LOG.info("assignedContainer" + " queue=" + getQueueName() + + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + + " cluster=" + cluster); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java index 83ee6c0..833c642 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java @@ -964,21 +964,20 @@ private void updateActiveUsersResourceUsage(String userName) { totalResUsageForActiveUsers.incUsed(partition, resourceUsage.getUsed(partition)); } - - if (LOG.isDebugEnabled()) { - LOG.debug("User '" + userName - + "' has become active. Hence move user to active list." - + "Active users size = " + activeUsersSet.size() - + "Non-active users size = " + nonActiveUsersSet.size() - + "Total Resource usage for active users=" - + totalResUsageForActiveUsers.getAllUsed() + "." - + "Total Resource usage for non-active users=" - + totalResUsageForNonActiveUsers.getAllUsed()); - } } } finally { this.writeLock.unlock(); } + if (LOG.isDebugEnabled()) { + LOG.debug("User '" + userName + + "' has become active. Hence move user to active list." + + "Active users size = " + activeUsersSet.size() + + "Non-active users size = " + nonActiveUsersSet.size() + + "Total Resource usage for active users=" + + totalResUsageForActiveUsers.getAllUsed() + "." + + "Total Resource usage for non-active users=" + + totalResUsageForNonActiveUsers.getAllUsed()); + } } private void updateNonActiveUsersResourceUsage(String userName) { @@ -1004,22 +1003,21 @@ private void updateNonActiveUsersResourceUsage(String userName) { resourceUsage.getUsed(partition)); totalResUsageForNonActiveUsers.incUsed(partition, resourceUsage.getUsed(partition)); - - if (LOG.isDebugEnabled()) { - LOG.debug("User '" + userName - + "' has become non-active.Hence move user to non-active list." - + "Active users size = " + activeUsersSet.size() - + "Non-active users size = " + nonActiveUsersSet.size() - + "Total Resource usage for active users=" - + totalResUsageForActiveUsers.getAllUsed() + "." - + "Total Resource usage for non-active users=" - + totalResUsageForNonActiveUsers.getAllUsed()); - } } } } finally { this.writeLock.unlock(); } + if (LOG.isDebugEnabled()) { + LOG.debug("User '" + userName + + "' has become non-active.Hence move user to non-active list." + + "Active users size = " + activeUsersSet.size() + + "Non-active users size = " + nonActiveUsersSet.size() + + "Total Resource usage for active users=" + + totalResUsageForActiveUsers.getAllUsed() + "." + + "Total Resource usage for non-active users=" + + totalResUsageForNonActiveUsers.getAllUsed()); + } } private ResourceUsage getTotalResourceUsagePerUser(String userName) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index 56a490b..65b4818 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -211,12 +211,12 @@ public Resource assignContainer(FSSchedulerNode node) { // empty before removal. Assigning an application to a queue and removal of // that queue both need the scheduler lock. TreeSet sortedChildQueues = new TreeSet<>(policy.getComparator()); + if (LOG.isDebugEnabled()) { + LOG.debug("Node " + node.getNodeName() + " offered to parent queue: " + + getName() + " visiting " + childQueues.size() + " children"); + } readLock.lock(); try { - if (LOG.isDebugEnabled()) { - LOG.debug("Node " + node.getNodeName() + " offered to parent queue: " + - getName() + " visiting " + childQueues.size() + " children"); - } sortedChildQueues.addAll(childQueues); for (FSQueue child : sortedChildQueues) { assigned = child.assignContainer(node); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 394ac59..5dd0b3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -667,10 +667,12 @@ private void removeApplication(ApplicationId applicationId, private void removeApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) { + + LOG.info("Application " + applicationAttemptId + " is done. finalState=" + + rmAppAttemptFinalState); + writeLock.lock(); try { - LOG.info("Application " + applicationAttemptId + " is done. finalState=" - + rmAppAttemptFinalState); FSAppAttempt attempt = getApplicationAttempt(applicationAttemptId); if (attempt == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java index 8adb2ed..edc7c1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java @@ -119,9 +119,9 @@ public void stop() { } public void applicationMasterFinished(ApplicationAttemptId appAttemptId) { + LOG.info("Application finished, removing password for " + appAttemptId); this.writeLock.lock(); try { - LOG.info("Application finished, removing password for " + appAttemptId); this.appAttemptSet.remove(appAttemptId); } finally { this.writeLock.unlock(); @@ -137,9 +137,9 @@ public void run() { @Private void rollMasterKey() { + LOG.info("Rolling master-key for amrm-tokens"); this.writeLock.lock(); try { - LOG.info("Rolling master-key for amrm-tokens"); this.nextMasterKey = createNewMasterKey(); AMRMTokenSecretManagerState state = AMRMTokenSecretManagerState.newInstance( @@ -190,9 +190,9 @@ public MasterKeyData createNewMasterKey() { public Token createAndGetAMRMToken( ApplicationAttemptId appAttemptId) { + LOG.info("Create AMRMToken for ApplicationAttempt: " + appAttemptId); this.writeLock.lock(); try { - LOG.info("Create AMRMToken for ApplicationAttempt: " + appAttemptId); AMRMTokenIdentifier identifier = new AMRMTokenIdentifier(appAttemptId, getMasterKey().getMasterKey() .getKeyId()); @@ -222,10 +222,10 @@ public MasterKeyData getMasterKey() { */ public void addPersistedPassword(Token token) throws IOException { + AMRMTokenIdentifier identifier = token.decodeIdentifier(); + LOG.debug("Adding password for " + identifier.getApplicationAttemptId()); this.writeLock.lock(); try { - AMRMTokenIdentifier identifier = token.decodeIdentifier(); - LOG.debug("Adding password for " + identifier.getApplicationAttemptId()); appAttemptSet.add(identifier.getApplicationAttemptId()); } finally { this.writeLock.unlock(); @@ -239,13 +239,13 @@ public void addPersistedPassword(Token token) @Override public byte[] retrievePassword(AMRMTokenIdentifier identifier) throws InvalidToken { + ApplicationAttemptId applicationAttemptId = + identifier.getApplicationAttemptId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to retrieve password for " + applicationAttemptId); + } this.readLock.lock(); try { - ApplicationAttemptId applicationAttemptId = - identifier.getApplicationAttemptId(); - if (LOG.isDebugEnabled()) { - LOG.debug("Trying to retrieve password for " + applicationAttemptId); - } if (!appAttemptSet.contains(applicationAttemptId)) { throw new InvalidToken(applicationAttemptId + " not found in AMRMTokenSecretManager."); @@ -300,11 +300,11 @@ public MasterKeyData getNextMasterKeyData() { @Override @Private protected byte[] createPassword(AMRMTokenIdentifier identifier) { + ApplicationAttemptId applicationAttemptId = + identifier.getApplicationAttemptId(); + LOG.info("Creating password for " + applicationAttemptId); this.readLock.lock(); try { - ApplicationAttemptId applicationAttemptId = - identifier.getApplicationAttemptId(); - LOG.info("Creating password for " + applicationAttemptId); return createPassword(identifier.getBytes(), getMasterKey() .getSecretKey()); } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java index 956391e..f342bb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java @@ -90,9 +90,9 @@ public NMTokenSecretManagerInRM(Configuration conf) { */ @Private public void rollMasterKey() { + LOG.info("Rolling master-key for nm-tokens"); super.writeLock.lock(); try { - LOG.info("Rolling master-key for nm-tokens"); if (this.currentMasterKey == null) { // Setting up for the first time. this.currentMasterKey = createNewMasterKey(); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java index 945d89e..fef93ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java @@ -104,9 +104,9 @@ public void stop() { */ @Private public void rollMasterKey() { + LOG.info("Rolling master-key for container-tokens"); super.writeLock.lock(); try { - LOG.info("Rolling master-key for container-tokens"); if (this.currentMasterKey == null) { // Setting up for the first time. this.currentMasterKey = createNewMasterKey(); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/VolumeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/VolumeImpl.java index 068e8a4..c965a8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/VolumeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/volume/csi/lifecycle/VolumeImpl.java @@ -183,19 +183,17 @@ public VolumeState transition(VolumeImpl volume, @Override public void handle(VolumeEvent event) { + VolumeId volumeId = event.getVolumeId(); + if (volumeId == null) { + // This should not happen, safely ignore the event + LOG.warn("Unexpected volume event received, event type is " + + event.getType().name() + ", but the volumeId is null."); + return; + } + LOG.info("Processing volume event, type=" + event.getType().name() + + ", volumeId=" + volumeId.toString()); try { this.writeLock.lock(); - VolumeId volumeId = event.getVolumeId(); - - if (volumeId == null) { - // This should not happen, safely ignore the event - LOG.warn("Unexpected volume event received, event type is " - + event.getType().name() + ", but the volumeId is null."); - return; - } - - LOG.info("Processing volume event, type=" + event.getType().name() - + ", volumeId=" + volumeId.toString()); VolumeState oldState = null; VolumeState newState = null; -- 2.7.4 (Apple Git-66)