From 09e5c2e337818f984980894d2e7dc575169472a8 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 22 Aug 2017 20:33:22 +0530 Subject: [PATCH] YARN-7039.YARN-3926 --- .../records/ApplicationResourceUsageReport.java | 1 - .../hadoop/yarn/api/records/ResourceTypeInfo.java | 6 +-- .../hadoop/yarn/api/records/impl/package-info.java | 4 ++ .../hadoop/yarn/util/UnitsConversionUtil.java | 5 +-- .../hadoop/yarn/util/resource/ResourceUtils.java | 4 +- .../hadoop/yarn/util/resource/package-info.java | 4 ++ .../hadoop/yarn/conf/TestResourceInformation.java | 3 ++ .../hadoop/yarn/util/TestUnitsConversionUtil.java | 8 +++- .../yarn/applications/distributedshell/Client.java | 20 ++++----- .../yarn/client/api/impl/AMRMClientImpl.java | 1 - .../pb/GetAllResourceProfilesResponsePBImpl.java | 5 --- .../pb/GetAllResourceTypeInfoRequestPBImpl.java | 3 +- .../impl/pb/GetResourceProfileRequestPBImpl.java | 5 --- .../impl/pb/GetResourceProfileResponsePBImpl.java | 5 --- .../pb/ApplicationResourceUsageReportPBImpl.java | 1 - .../records/impl/pb/ProfileCapabilityPBImpl.java | 5 --- .../records/impl/pb/ResourceTypeInfoPBImpl.java | 2 - .../hadoop/yarn/util/resource/Resources.java | 6 ++- .../apache/hadoop/yarn/api/TestPBImplRecords.java | 4 -- .../yarn/util/resource/TestResourceUtils.java | 47 ++++++++++++---------- .../server/nodemanager/NodeStatusUpdaterImpl.java | 8 ++-- .../nodemanager/util/NodeManagerHardwareUtils.java | 36 ++++++++--------- .../util/TestNodeManagerHardwareUtils.java | 2 +- .../resourcemanager/ApplicationMasterService.java | 1 - .../server/resourcemanager/ResourceManager.java | 2 +- .../resource/ResourceProfilesManagerImpl.java | 8 +++- .../resource/TestResourceProfiles.java | 8 ++-- 27 files changed, 100 insertions(+), 104 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index f9c8975..d2e33ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -24,7 +24,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; -import java.util.HashMap; import java.util.Map; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java index d85cf0c..b6f7f14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java @@ -19,9 +19,7 @@ package org.apache.hadoop.yarn.api.records; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.util.Records; @@ -111,7 +109,7 @@ public static ResourceTypeInfo newInstance(String name, String units, } /** - * Create a new instance of ResourceTypeInfo from name, units + * Create a new instance of ResourceTypeInfo from name, units. * * @param name name of resource type * @param units units of resource type @@ -124,7 +122,7 @@ public static ResourceTypeInfo newInstance(String name, String units) { } /** - * Create a new instance of ResourceTypeInfo from name + * Create a new instance of ResourceTypeInfo from name. * * @param name name of resource type * @return the new ResourceTypeInfo object diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java index d04e339..b2420bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java @@ -15,6 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/** + * Package org.apache.hadoop.yarn.api.records.impl contains classes + * which define basic resources. + */ @InterfaceAudience.Public @InterfaceStability.Unstable package org.apache.hadoop.yarn.api.records.impl; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java index 7b737bc..7a212e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java @@ -45,9 +45,8 @@ } } - private static final String[] UNITS = - { "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti", - "Pi" }; + private static final String[] UNITS = {"p", "n", "u", "m", "", "k", "M", "G", + "T", "P", "Ki", "Mi", "Gi", "Ti", "Pi"}; private static final List SORTED_UNITS = Arrays.asList(UNITS); public static final Set KNOWN_UNITS = createKnownUnitsSet(); private static final Converter PICO = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index b81c327..ed16104 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -466,8 +466,8 @@ public static String getUnits(String resourceValue) { return readOnlyNodeResources; } - private static Map - initializeNodeResourceInformation(Configuration conf) { + private static Map initializeNodeResourceInformation( + Configuration conf) { Map nodeResources = new HashMap<>(); try { addResourcesFileToConf( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java index 01c7eb0..1e925d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java @@ -15,6 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/** + * Package org.apache.hadoop.yarn.util.resource contains classes + * which is used as utility class for resource profile computations. + */ @InterfaceAudience.Public @InterfaceStability.Unstable package org.apache.hadoop.yarn.util.resource; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java index 2efb990..66bf320 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java @@ -22,6 +22,9 @@ import org.junit.Assert; import org.junit.Test; +/** + * Test class to verify various resource informations in a given resource. + */ public class TestResourceInformation { @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java index 0b512a5..a412fae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java @@ -21,6 +21,10 @@ import org.junit.Assert; import org.junit.Test; +/** + * Test class to handle all test cases needed to verify basic unit conversion + * scenarios. + */ public class TestUnitsConversionUtil { @Test @@ -87,13 +91,13 @@ public void testOverflow() { UnitsConversionUtil.convert("P", "p", test); Assert.fail("this operation should result in an overflow"); } catch (IllegalArgumentException ie) { - ; // do nothing + // do nothing } try { UnitsConversionUtil.convert("m", "p", Long.MAX_VALUE - 1); Assert.fail("this operation should result in an overflow"); } catch (IllegalArgumentException ie) { - ; // do nothing + // do nothing } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 0c6d2d3..6535078 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -81,7 +81,7 @@ import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.ResourceProfilesNotEnabledException; +import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; @@ -553,7 +553,7 @@ public boolean run() throws IOException, YarnException { Map profiles; try { profiles = yarnClient.getResourceProfiles(); - } catch (ResourceProfilesNotEnabledException re) { + } catch (YARNFeatureNotEnabledException re) { profiles = null; } @@ -994,15 +994,17 @@ private void setAMResourceCapability(ApplicationSubmissionContext appContext, if (profile.isEmpty()) { tmp = "default"; } - if (appContext.getAMContainerResourceRequest() == null) { - appContext.setAMContainerResourceRequest(ResourceRequest - .newInstance(Priority.newInstance(priority), "*", + if (appContext.getAMContainerResourceRequests().get(0) == null) { + List amResourceRequests = new ArrayList(); + amResourceRequests + .add(ResourceRequest.newInstance(Priority.newInstance(priority), "*", Resources.clone(Resources.none()), 1)); + appContext.setAMContainerResourceRequests(amResourceRequests); } - if (appContext.getAMContainerResourceRequest().getProfileCapability() - == null) { - appContext.getAMContainerResourceRequest().setProfileCapability( + if (appContext.getAMContainerResourceRequests().get(0) + .getProfileCapability() == null) { + appContext.getAMContainerResourceRequests().get(0).setProfileCapability( ProfileCapability.newInstance(tmp, Resource.newInstance(0, 0))); } Resource capability = Resource.newInstance(0, 0); @@ -1018,7 +1020,7 @@ private void setAMResourceCapability(ApplicationSubmissionContext appContext, capability.setMemorySize(memory); capability.setVirtualCores(vcores); } - appContext.getAMContainerResourceRequest().getProfileCapability() + appContext.getAMContainerResourceRequests().get(0).getProfileCapability() .setProfileCapabilityOverride(capability); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java index 8e66c20..a41ab6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; -import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java index bc4a9b6..eaa392f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java @@ -134,9 +134,4 @@ private ResourceProto convertToProtoFormat(Resource res) { r.setVirtualCores(res.getVirtualCores()); return r.getProto(); } - - @Override - public int hashCode() { - return getProto().hashCode(); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java index b43a3b1..b3f4692 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java @@ -60,8 +60,9 @@ public int hashCode() { @Override public boolean equals(Object other) { - if (other == null) + if (other == null) { return false; + } if (other.getClass().isAssignableFrom(this.getClass())) { return this.getProto().equals(this.getClass().cast(other).getProto()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileRequestPBImpl.java index 926680b..89a680c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileRequestPBImpl.java @@ -93,9 +93,4 @@ public String getProfileName() { } return profile; } - - @Override - public int hashCode() { - return getProto().hashCode(); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java index b82eaac..e08d077 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java @@ -104,9 +104,4 @@ private void maybeInitBuilder() { } viaProto = false; } - - @Override - public int hashCode() { - return getProto().hashCode(); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java index 2f94a24..14ede5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java @@ -29,7 +29,6 @@ import com.google.protobuf.TextFormat; -import java.util.HashMap; import java.util.Map; @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java index d6d5643..8c161f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java @@ -126,9 +126,4 @@ private void maybeInitBuilder() { r.setVirtualCores(res.getVirtualCores()); return r.getProto(); } - - @Override - public int hashCode() { - return getProto().hashCode(); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java index 0454b96..17230e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java @@ -26,8 +26,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypeInfoProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypesProto; -import com.google.common.base.Preconditions; - /** * {@code ResourceTypeInfoPBImpl} which implements the * {@link ResourceTypeInfo} class which represents different resource types diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 54ef036..1e2ce15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -30,6 +30,10 @@ import java.util.Arrays; +/** + * Resources is a computation class which provides a set of apis to do + * mathematical operations on Resource object. + */ @InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" }) @Unstable public class Resources { @@ -47,7 +51,7 @@ private String name; /** - * Constructor for a fixed value resource + * Constructor for a fixed value resource. * @param rName the name of the resource * @param value the fixed value to be returned for all resource types */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 164880b..722a4f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -29,8 +29,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest; @@ -146,7 +144,6 @@ import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceAllocationRequest; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceOption; @@ -349,7 +346,6 @@ import org.junit.Test; import com.google.common.collect.ImmutableSet; -import java.io.IOException; import java.util.Arrays; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 4e4671a..a5550a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -33,6 +33,9 @@ import java.util.HashMap; import java.util.Map; +/** + * Test class to verify all resource utility methods. + */ public class TestResourceUtils { static class ResourceFileInformation { @@ -106,8 +109,8 @@ public void testGetResourceTypesConfigs() throws Exception { testFile4.resourceNameUnitsMap.put("resource1", "G"); testFile4.resourceNameUnitsMap.put("resource2", "m"); - ResourceFileInformation[] tests = - { testFile1, testFile2, testFile3, testFile4 }; + ResourceFileInformation[] tests = {testFile1, testFile2, testFile3, + testFile4}; Map res; for (ResourceFileInformation testInformation : tests) { ResourceUtils.resetResourceTypes(); @@ -134,9 +137,9 @@ public void testGetResourceTypesConfigs() throws Exception { public void testGetResourceTypesConfigErrors() throws Exception { Configuration conf = new YarnConfiguration(); - String[] resourceFiles = - { "resource-types-error-1.xml", "resource-types-error-2.xml", - "resource-types-error-3.xml", "resource-types-error-4.xml" }; + String[] resourceFiles = {"resource-types-error-1.xml", + "resource-types-error-2.xml", "resource-types-error-3.xml", + "resource-types-error-4.xml"}; for (String resourceFile : resourceFiles) { ResourceUtils.resetResourceTypes(); File dest = null; @@ -159,15 +162,15 @@ public void testGetResourceTypesConfigErrors() throws Exception { @Test public void testInitializeResourcesMap() throws Exception { - String[] empty = { "", "" }; - String[] res1 = { "resource1", "m" }; - String[] res2 = { "resource2", "G" }; - String[][] test1 = { empty }; - String[][] test2 = { res1 }; - String[][] test3 = { res2 }; - String[][] test4 = { res1, res2 }; + String[] empty = {"", ""}; + String[] res1 = {"resource1", "m"}; + String[] res2 = {"resource2", "G"}; + String[][] test1 = {empty}; + String[][] test2 = {res1}; + String[][] test3 = {res2}; + String[][] test4 = {res1, res2}; - String[][][] allTests = { test1, test2, test3, test4 }; + String[][][] allTests = {test1, test2, test3, test4}; for (String[][] test : allTests) { @@ -221,19 +224,19 @@ public void testInitializeResourcesMap() throws Exception { @Test public void testInitializeResourcesMapErrors() throws Exception { - String[] mem1 = { "memory-mb", "" }; - String[] vcores1 = { "vcores", "M" }; + String[] mem1 = {"memory-mb", ""}; + String[] vcores1 = {"vcores", "M"}; - String[] mem2 = { "memory-mb", "m" }; - String[] vcores2 = { "vcores", "G" }; + String[] mem2 = {"memory-mb", "m"}; + String[] vcores2 = {"vcores", "G"}; - String[] mem3 = { "memory", "" }; + String[] mem3 = {"memory", ""}; - String[][] test1 = { mem1, vcores1 }; - String[][] test2 = { mem2, vcores2 }; - String[][] test3 = { mem3 }; + String[][] test1 = {mem1, vcores1}; + String[][] test2 = {mem2, vcores2}; + String[][] test3 = {mem3}; - String[][][] allTests = { test1, test2, test3 }; + String[][][] allTests = {test1, test2, test3}; for (String[][] test : allTests) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index d085b29..aaf6764 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -174,12 +174,12 @@ public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher, @Override protected void serviceInit(Configuration conf) throws Exception { this.totalResource = NodeManagerHardwareUtils.getNodeResources(conf); - int memoryMb = totalResource.getMemory(); + long memoryMb = totalResource.getMemorySize(); float vMemToPMem = conf.getFloat( YarnConfiguration.NM_VMEM_PMEM_RATIO, YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); - int virtualMemoryMb = (int)Math.ceil(memoryMb * vMemToPMem); + long virtualMemoryMb = (long)Math.ceil(memoryMb * vMemToPMem); int virtualCores = totalResource.getVirtualCores(); LOG.info("Nodemanager resources: memory set to " + memoryMb + "MB."); @@ -189,12 +189,12 @@ protected void serviceInit(Configuration conf) throws Exception { metrics.addResource(totalResource); // Get actual node physical resources - int physicalMemoryMb = memoryMb; + long physicalMemoryMb = memoryMb; int physicalCores = virtualCores; ResourceCalculatorPlugin rcp = ResourceCalculatorPlugin.getNodeResourceMonitorPlugin(conf); if (rcp != null) { - physicalMemoryMb = (int) (rcp.getPhysicalMemorySize() / (1024 * 1024)); + physicalMemoryMb = rcp.getPhysicalMemorySize() / (1024 * 1024); physicalCores = rcp.getNumProcessors(); } this.physicalResource = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java index 3057770..c52ac1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java @@ -246,8 +246,8 @@ private static int getVCoresInternal(ResourceCalculatorPlugin plugin, return cores; } - private static int getConfiguredMemoryMB(Configuration conf) { - int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, + private static long getConfiguredMemoryMB(Configuration conf) { + long memoryMb = conf.getLong(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB); if (memoryMb == -1) { memoryMb = YarnConfiguration.DEFAULT_NM_PMEM_MB; @@ -270,7 +270,7 @@ private static int getConfiguredMemoryMB(Configuration conf) { * - the configuration for the NodeManager * @return the amount of memory that will be used for YARN containers in MB. */ - public static int getContainerMemoryMB(Configuration conf) { + public static long getContainerMemoryMB(Configuration conf) { if (!isHardwareDetectionEnabled(conf)) { return getConfiguredMemoryMB(conf); } @@ -299,7 +299,7 @@ public static int getContainerMemoryMB(Configuration conf) { * - the configuration for the NodeManager * @return the amount of memory that will be used for YARN containers in MB. */ - public static int getContainerMemoryMB(ResourceCalculatorPlugin plugin, + public static long getContainerMemoryMB(ResourceCalculatorPlugin plugin, Configuration conf) { if (!isHardwareDetectionEnabled(conf) || plugin == null) { return getConfiguredMemoryMB(conf); @@ -307,26 +307,24 @@ public static int getContainerMemoryMB(ResourceCalculatorPlugin plugin, return getContainerMemoryMBInternal(plugin, conf); } - private static int getContainerMemoryMBInternal(ResourceCalculatorPlugin plugin, + private static long getContainerMemoryMBInternal(ResourceCalculatorPlugin plugin, Configuration conf) { - int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, -1); + long memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, -1); if (memoryMb == -1) { - int physicalMemoryMB = - (int) (plugin.getPhysicalMemorySize() / (1024 * 1024)); - int hadoopHeapSizeMB = - (int) (Runtime.getRuntime().maxMemory() / (1024 * 1024)); - int containerPhysicalMemoryMB = - (int) (0.8f * (physicalMemoryMB - (2 * hadoopHeapSizeMB))); - int reservedMemoryMB = - conf.getInt(YarnConfiguration.NM_SYSTEM_RESERVED_PMEM_MB, -1); + long physicalMemoryMB = (plugin.getPhysicalMemorySize() / (1024 * 1024)); + long hadoopHeapSizeMB = (Runtime.getRuntime().maxMemory() + / (1024 * 1024)); + long containerPhysicalMemoryMB = (long) (0.8f + * (physicalMemoryMB - (2 * hadoopHeapSizeMB))); + long reservedMemoryMB = conf + .getInt(YarnConfiguration.NM_SYSTEM_RESERVED_PMEM_MB, -1); if (reservedMemoryMB != -1) { containerPhysicalMemoryMB = physicalMemoryMB - reservedMemoryMB; } - if(containerPhysicalMemoryMB <= 0) { + if (containerPhysicalMemoryMB <= 0) { LOG.error("Calculated memory for YARN containers is too low." + " Node memory is " + physicalMemoryMB - + " MB, system reserved memory is " - + reservedMemoryMB + " MB."); + + " MB, system reserved memory is " + reservedMemoryMB + " MB."); } containerPhysicalMemoryMB = Math.max(containerPhysicalMemoryMB, 0); memoryMb = containerPhysicalMemoryMB; @@ -365,8 +363,8 @@ public static Resource getNodeResources(Configuration configuration) { } ResourceInformation memResInfo = resourceInformation.get(memory); if(memResInfo.getValue() == 0) { - ret.setMemory(getContainerMemoryMB(conf)); - LOG.debug("Set memory to " + ret.getMemory()); + ret.setMemorySize(getContainerMemoryMB(conf)); + LOG.debug("Set memory to " + ret.getMemorySize()); } } if (resourceInformation.containsKey(vcores)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java index 4add586..767c308 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java @@ -172,7 +172,7 @@ public void testGetContainerMemoryMB() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, true); - int mem = NodeManagerHardwareUtils.getContainerMemoryMB(null, conf); + long mem = NodeManagerHardwareUtils.getContainerMemoryMB(null, conf); Assert.assertEquals(YarnConfiguration.DEFAULT_NM_PMEM_MB, mem); mem = NodeManagerHardwareUtils.getContainerMemoryMB(plugin, conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index ccd984a..f77d09e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -60,7 +60,6 @@ import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index e478dc2..e53a42c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -239,7 +239,7 @@ protected void serviceInit(Configuration conf) throws Exception { new ResourceProfilesManagerImpl(); resourceProfilesManager.init(conf); rmContext.setResourceProfilesManager(resourceProfilesManager); - + this.configurationProvider = ConfigurationProviderFactory.getConfigurationProvider(conf); this.configurationProvider.init(this.conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java index e213a97..dfd14e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java @@ -45,6 +45,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; +/** + * PBImpl class to handle all proto related implementation for + * ResourceProfilesManager. + */ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { private static final Log LOG = @@ -66,8 +70,8 @@ protected final ReentrantReadWriteLock.ReadLock readLock; protected final ReentrantReadWriteLock.WriteLock writeLock; - private static final String[] MANDATORY_PROFILES = - { DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE }; + private static final String[] MANDATORY_PROFILES = {DEFAULT_PROFILE, + MINIMUM_PROFILE, MAXIMUM_PROFILE}; private static final String FEATURE_NOT_ENABLED_MSG = "Resource profile is not enabled, please " + "enable resource profile feature before using its functions." diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java index b53e59c..a38f59b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java @@ -29,6 +29,9 @@ import java.util.HashMap; import java.util.Map; +/** + * Common test class for resource profile related tests. + */ public class TestResourceProfiles { @Test @@ -86,9 +89,8 @@ public void testLoadProfilesMissingMandatoryProfile() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true); - String[] badProfiles = { "profiles/illegal-profiles-1.json", - "profiles/illegal-profiles-2.json", - "profiles/illegal-profiles-3.json" }; + String[] badProfiles = {"profiles/illegal-profiles-1.json", + "profiles/illegal-profiles-2.json", "profiles/illegal-profiles-3.json"}; for (String file : badProfiles) { ResourceProfilesManager manager = new ResourceProfilesManagerImpl(); conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE, file); -- 2.10.1 (Apple Git-78)