diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index fce78c9..8cbfdfa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -323,47 +323,6 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED_DEFAULT = false; - /** Minimum memory (in MB) used for allocating an opportunistic container. */ - public static final String OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB = - YARN_PREFIX + "opportunistic-containers.min-memory-mb"; - public static final int OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT = 512; - - /** Minimum virtual CPU cores used for allocating an opportunistic container. - * */ - public static final String OPPORTUNISTIC_CONTAINERS_MIN_VCORES = - YARN_PREFIX + "opportunistic-containers.min-vcores"; - public static final int OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT = 1; - - /** Maximum memory (in MB) used for allocating an opportunistic container. */ - public static final String OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB = - YARN_PREFIX + "opportunistic-containers.max-memory-mb"; - public static final int OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT = 2048; - - /** Maximum virtual CPU cores used for allocating an opportunistic container. - * */ - public static final String OPPORTUNISTIC_CONTAINERS_MAX_VCORES = - YARN_PREFIX + "opportunistic-containers.max-vcores"; - public static final int OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT = 4; - - /** Incremental memory (in MB) used for allocating an opportunistic container. - * */ - public static final String OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB = - YARN_PREFIX + "opportunistic-containers.incr-memory-mb"; - public static final int OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT = - 512; - - /** Incremental virtual CPU cores used for allocating an opportunistic - * container. */ - public static final String OPPORTUNISTIC_CONTAINERS_INCR_VCORES = - YARN_PREFIX + "opportunistic-containers.incr-vcores"; - public static final int OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT = 1; - - /** Container token expiry for opportunistic containers. */ - public static final String OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS = - YARN_PREFIX + "opportunistic-containers.container-token-expiry-ms"; - public static final int OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT = - 600000; - /** Number of nodes to be used by the Opportunistic Container allocator for * dispatching containers during container allocation. */ public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterDistributedSchedulingAMResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterDistributedSchedulingAMResponse.java index f7d8df2..70483f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterDistributedSchedulingAMResponse.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterDistributedSchedulingAMResponse.java @@ -74,14 +74,6 @@ public abstract void setRegisterResponse( @Public @Unstable - public abstract void setIncrContainerResource(Resource maxResource); - - @Public - @Unstable - public abstract Resource getIncrContainerResource(); - - @Public - @Unstable public abstract void setContainerTokenExpiryInterval(int interval); @Public diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterDistributedSchedulingAMResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterDistributedSchedulingAMResponsePBImpl.java index 41b2a4f..f52c03e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterDistributedSchedulingAMResponsePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterDistributedSchedulingAMResponsePBImpl.java @@ -53,7 +53,6 @@ private Resource maxContainerResource; private Resource minContainerResource; - private Resource incrContainerResource; private List nodesForScheduling; private RegisterApplicationMasterResponse registerApplicationMasterResponse; @@ -109,10 +108,6 @@ private synchronized void mergeLocalToBuilder() { builder.setMinContainerResource(ProtoUtils.convertToProtoFormat( this.minContainerResource)); } - if (this.incrContainerResource != null) { - builder.setIncrContainerResource( - ProtoUtils.convertToProtoFormat(this.incrContainerResource)); - } if (this.registerApplicationMasterResponse != null) { builder.setRegisterResponse( ((RegisterApplicationMasterResponsePBImpl) @@ -202,33 +197,6 @@ public Resource getMinContainerResource() { } @Override - public void setIncrContainerResource(Resource incrResource) { - maybeInitBuilder(); - if (incrContainerResource == null) { - builder.clearIncrContainerResource(); - } - this.incrContainerResource = incrResource; - } - - @Override - public Resource getIncrContainerResource() { - if (this.incrContainerResource != null) { - return this.incrContainerResource; - } - - YarnServerCommonServiceProtos. - RegisterDistributedSchedulingAMResponseProtoOrBuilder p = - viaProto ? proto : builder; - if (!p.hasIncrContainerResource()) { - return null; - } - - this.incrContainerResource = ProtoUtils.convertFromProtoFormat(p - .getIncrContainerResource()); - return this.incrContainerResource; - } - - @Override public void setContainerTokenExpiryInterval(int interval) { maybeInitBuilder(); builder.setContainerTokenExpiryInterval(interval); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto index edb2d9c..0896888 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto @@ -35,10 +35,9 @@ message RegisterDistributedSchedulingAMResponseProto { optional RegisterApplicationMasterResponseProto register_response = 1; optional ResourceProto max_container_resource = 2; optional ResourceProto min_container_resource = 3; - optional ResourceProto incr_container_resource = 4; - optional int32 container_token_expiry_interval = 5; - optional int64 container_id_start = 6; - repeated RemoteNodeProto nodes_for_scheduling = 7; + optional int32 container_token_expiry_interval = 4; + optional int64 container_id_start = 5; + repeated RemoteNodeProto nodes_for_scheduling = 6; } message DistributedSchedulingAllocateResponseProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java index 0c2b1ac..98c7b28 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java @@ -170,11 +170,11 @@ public OpportunisticContainersStatus getOpportunisticContainersStatus() { this.opportunisticContainersStatus.setWaitQueueLength( getNumQueuedContainers()); this.opportunisticContainersStatus.setOpportMemoryUsed( - metrics.getOpportMemoryUsed()); + metrics.getAllocatedOpportunisticGB()); this.opportunisticContainersStatus.setOpportCoresUsed( - metrics.getOpportCoresUsed()); + metrics.getAllocatedOpportunisticVCores()); this.opportunisticContainersStatus.setRunningOpportContainers( - metrics.getRunningOpportContainers()); + metrics.getRunningOpportunisticContainers()); return this.opportunisticContainersStatus; } @@ -196,7 +196,7 @@ private void onContainerCompleted(Container container) { this.utilizationTracker.subtractContainerResource(container); if (container.getContainerTokenIdentifier().getExecutionType() == ExecutionType.OPPORTUNISTIC) { - this.metrics.opportunisticContainerCompleted(container); + this.metrics.opportunisticContainerCompleted(container.getResource()); } startPendingContainers(); } @@ -298,7 +298,7 @@ private void startAllocatedContainer(Container container) { this.utilizationTracker.addContainerResources(container); if (container.getContainerTokenIdentifier().getExecutionType() == ExecutionType.OPPORTUNISTIC) { - this.metrics.opportunisticContainerStarted(container); + this.metrics.startOpportunisticContainer(container.getResource()); } container.sendLaunchEvent(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java index b001b63..7c0e8ac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java @@ -64,12 +64,12 @@ @Metric("Disk utilization % on good log dirs") MutableGaugeInt goodLogDirsDiskUtilizationPerc; - @Metric("Memory used by Opportunistic Containers in MB") - MutableGaugeLong opportMemoryUsed; - @Metric("# of Virtual Cores used by opportunistic containers") - MutableGaugeInt opportCoresUsed; + @Metric("Current allocated memory by opportunistic containers in GB") + MutableGaugeLong allocatedOpportunisticGB; + @Metric("Current allocated Virtual Cores by opportunistic containers") + MutableGaugeInt allocatedOpportunisticVCores; @Metric("# of running opportunistic containers") - MutableGaugeInt runningOpportContainers; + MutableGaugeInt runningOpportunisticContainers; // CHECKSTYLE:ON:VisibilityModifier @@ -77,6 +77,7 @@ private long allocatedMB; private long availableMB; + private long allocatedOpportunisticMB; public NodeManagerMetrics(JvmMetrics jvmMetrics) { this.jvmMetrics = jvmMetrics; @@ -141,30 +142,6 @@ public void endReInitingContainer() { containersReIniting.decr(); } - public long getOpportMemoryUsed() { - return opportMemoryUsed.value(); - } - - public int getOpportCoresUsed() { - return opportCoresUsed.value(); - } - - public int getRunningOpportContainers() { - return runningOpportContainers.value(); - } - - public void opportunisticContainerCompleted(Container container) { - opportMemoryUsed.decr(container.getResource().getMemorySize()); - opportCoresUsed.decr(container.getResource().getVirtualCores()); - runningOpportContainers.decr(); - } - - public void opportunisticContainerStarted(Container container) { - opportMemoryUsed.incr(container.getResource().getMemorySize()); - opportCoresUsed.incr(container.getResource().getVirtualCores()); - runningOpportContainers.incr(); - } - public void allocateContainer(Resource res) { allocatedContainers.incr(); allocatedMB = allocatedMB + res.getMemorySize(); @@ -196,6 +173,22 @@ public void changeContainer(Resource before, Resource now) { availableVCores.decr(deltaVCores); } + public void startOpportunisticContainer(Resource res) { + runningOpportunisticContainers.incr(); + allocatedOpportunisticMB = allocatedOpportunisticMB + res.getMemorySize(); + allocatedOpportunisticGB + .set((int) Math.ceil(allocatedOpportunisticMB / 1024d)); + allocatedOpportunisticVCores.incr(res.getVirtualCores()); + } + + public void opportunisticContainerCompleted(Resource res) { + runningOpportunisticContainers.decr(); + allocatedOpportunisticMB = allocatedOpportunisticMB - res.getMemorySize(); + allocatedOpportunisticGB + .set((int) Math.ceil(allocatedOpportunisticMB / 1024d)); + allocatedOpportunisticVCores.decr(res.getVirtualCores()); + } + public void addResource(Resource res) { availableMB = availableMB + res.getMemorySize(); availableGB.incr((int)Math.floor(availableMB/1024d)); @@ -272,4 +265,16 @@ public int getReInitializingContainer() { public int getContainersRolledbackOnFailure() { return containersRolledBackOnFailure.value(); } + + public long getAllocatedOpportunisticGB() { + return allocatedOpportunisticGB.value(); + } + + public int getAllocatedOpportunisticVCores() { + return allocatedOpportunisticVCores.value(); + } + + public int getRunningOpportunisticContainers() { + return runningOpportunisticContainers.value(); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/DistributedScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/DistributedScheduler.java index 0f47c93..2f2c624 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/DistributedScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/DistributedScheduler.java @@ -184,14 +184,10 @@ private void updateAllocateResponse(AllocateResponse response, private void updateParameters( RegisterDistributedSchedulingAMResponse registerResponse) { - Resource incrementResource = registerResponse.getIncrContainerResource(); - if (incrementResource == null) { - incrementResource = registerResponse.getMinContainerResource(); - } oppContainerContext.updateAllocationParams( registerResponse.getMinContainerResource(), registerResponse.getMaxContainerResource(), - incrementResource, + registerResponse.getMinContainerResource(), registerResponse.getContainerTokenExpiryInterval()); oppContainerContext.getContainerIdGenerator() diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java index 7814b84..12e9d02 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java @@ -199,9 +199,8 @@ public long generateContainerId() { } }); int tokenExpiryInterval = getConfig() - .getInt(YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT); + .getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); opCtx.updateAllocationParams(createMinContainerResource(), createMaxContainerResource(), createIncrContainerResource(), tokenExpiryInterval); @@ -275,12 +274,10 @@ public AllocateResponse allocate(AllocateRequest request) throws dsResp.setRegisterResponse(response); dsResp.setMinContainerResource(createMinContainerResource()); dsResp.setMaxContainerResource(createMaxContainerResource()); - dsResp.setIncrContainerResource(createIncrContainerResource()); dsResp.setContainerTokenExpiryInterval( getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT)); + YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS)); dsResp.setContainerIdStart( this.rmContext.getEpoch() << ResourceManager.EPOCH_BIT_SHIFT); @@ -384,18 +381,6 @@ public QueueLimitCalculator getNodeManagerQueueLimitCalculator() { return nodeMonitor.getThresholdCalculator(); } - private Resource createIncrContainerResource() { - return Resource.newInstance( - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT), - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES, - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT) - ); - } - private synchronized List getLeastLoadedNodes() { long currTime = System.currentTimeMillis(); if ((currTime - lastCacheUpdateTime > cacheRefreshInterval) @@ -428,27 +413,29 @@ private RemoteNode convertToRemoteNode(NodeId nodeId) { private Resource createMaxContainerResource() { return Resource.newInstance( getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB, - YarnConfiguration - .OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT), + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB), getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES, - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT) + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES) ); } private Resource createMinContainerResource() { return Resource.newInstance( getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT), + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB), getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES, - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT) + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES) ); } + private Resource createIncrContainerResource() { + return createMinContainerResource(); + } + private static ApplicationAttemptId getAppAttemptId() throws YarnException { AMRMTokenIdentifier amrmTokenIdentifier = YarnServerSecurityUtils.authorizeRequest(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index 995a7b0..6744c2e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -149,10 +149,10 @@ public String getRackName() { */ public synchronized void allocateContainer(RMContainer rmContainer) { Container container = rmContainer.getContainer(); - if (rmContainer.getExecutionType() != ExecutionType.OPPORTUNISTIC) { + if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { deductUnallocatedResource(container.getResource()); + ++numContainers; } - ++numContainers; launchedContainers.put(container.getId(), rmContainer); @@ -251,8 +251,8 @@ protected synchronized void updateResourceForReleasedContainer( Container container) { if (container.getExecutionType() == ExecutionType.GUARANTEED) { addUnallocatedResource(container.getResource()); + --numContainers; } - --numContainers; } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index 7b68b84..c03df63 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -49,6 +49,7 @@ static class NodesBlock extends HtmlBlock { final ResourceManager rm; private static final long BYTES_IN_MB = 1024 * 1024; + private static final long BYTES_IN_GB = 1024 * 1024 * 1024; private static boolean opportunisticContainersEnabled; @Inject @@ -181,8 +182,9 @@ protected void render(Block html) { nodeTableData .append(String.valueOf(info.getNumRunningOpportContainers())) .append("\",\"").append("
") - .append(StringUtils.byteDesc(info.getUsedMemoryOpport())) + .append(String.valueOf(info.getUsedMemoryOpportGB())).append("'>") + .append(StringUtils.byteDesc( + info.getUsedMemoryOpportGB() * BYTES_IN_GB)) .append("\",\"") .append(String.valueOf(info.getUsedVirtualCoresOpport())) .append("\",\"") diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index aee7717..3416e52 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -51,7 +51,7 @@ protected long usedVirtualCores; protected long availableVirtualCores; private int numRunningOpportContainers; - private long usedMemoryOpport; // Memory in bytes. + private long usedMemoryOpportGB; private long usedVirtualCoresOpport; private int numQueuedContainers; protected ArrayList nodeLabels = new ArrayList(); @@ -85,7 +85,7 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) { // Status of opportunistic containers. this.numRunningOpportContainers = 0; - this.usedMemoryOpport = 0; + this.usedMemoryOpportGB = 0; this.usedVirtualCoresOpport = 0; this.numQueuedContainers = 0; OpportunisticContainersStatus opportStatus = @@ -93,7 +93,7 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) { if (opportStatus != null) { this.numRunningOpportContainers = opportStatus.getRunningOpportContainers(); - this.usedMemoryOpport = opportStatus.getOpportMemoryUsed(); + this.usedMemoryOpportGB = opportStatus.getOpportMemoryUsed(); this.usedVirtualCoresOpport = opportStatus.getOpportCoresUsed(); this.numQueuedContainers = opportStatus.getQueuedOpportContainers(); } @@ -165,8 +165,8 @@ public int getNumRunningOpportContainers() { return numRunningOpportContainers; } - public long getUsedMemoryOpport() { - return usedMemoryOpport; + public long getUsedMemoryOpportGB() { + return usedMemoryOpportGB; } public long getUsedVirtualCoresOpport() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java index 73d9e5c..06518aa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java @@ -273,8 +273,6 @@ public RMContainerTokenSecretManager getContainerTokenSecretManager() { dsRegResp.getMaxContainerResource().getVirtualCores()); Assert.assertEquals(1024, dsRegResp.getMinContainerResource().getMemorySize()); - Assert.assertEquals(2, - dsRegResp.getIncrContainerResource().getVirtualCores()); DistributedSchedulingAllocateRequestPBImpl distAllReq = (DistributedSchedulingAllocateRequestPBImpl)factory.newRecordInstance( @@ -344,7 +342,6 @@ public AllocateResponse allocate(AllocateRequest request) throws resp.setContainerIdStart(54321L); resp.setMaxContainerResource(Resource.newInstance(4096, 4)); resp.setMinContainerResource(Resource.newInstance(1024, 1)); - resp.setIncrContainerResource(Resource.newInstance(2048, 2)); return resp; }