diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java
index b7d925a6592..af79347175e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java
@@ -27,6 +27,7 @@
.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import java.io.IOException;
@@ -52,11 +53,13 @@ void init(ApplicationMasterServiceContext amsContext,
* @param request Register Request.
* @param response Register Response.
* @throws IOException IOException.
+ * @throws YarnException in critical situation where invalid
+ * profiles/resources are added.
*/
- void registerApplicationMaster(
- ApplicationAttemptId applicationAttemptId,
+ void registerApplicationMaster(ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException;
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException;
/**
* Allocate call.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
index e745697c4bb..8456a8ec59f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
@@ -81,6 +81,7 @@
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
/**
*
The protocol between clients and the ResourceManager
@@ -604,7 +605,8 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts(
* @param request request to get all the resource profiles
* @return Response containing a map of the profile name to Resource
* capabilities
- * @throws YarnException if resource profiles are not enabled on the RM
+ * @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
@@ -618,8 +620,8 @@ GetAllResourceProfilesResponse getResourceProfiles(
*
* @param request request to get the details of a resource profile
* @return Response containing the details for a particular resource profile
- * @throws YarnException if resource profiles are not enabled on the RM or
- * the profile cannot be found
+ * @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
@@ -633,8 +635,7 @@ GetResourceProfileResponse getResourceProfile(
*
* @param request request to get the details of a resource profile
* @return Response containing the details for a particular resource profile
- * @throws YarnException if resource profiles are not enabled on the RM or
- * the profile cannot be found
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java
new file mode 100644
index 00000000000..62340fea363
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * This exception is thrown when a feature is being used which is not enabled
+ * yet.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class YARNFeatureNotEnabledException extends YarnException {
+ private static final long serialVersionUID = 898023752676L;
+
+ public YARNFeatureNotEnabledException(Throwable cause) {
+ super(cause);
+ }
+
+ public YARNFeatureNotEnabledException(String message) {
+ super(message);
+ }
+
+ public YARNFeatureNotEnabledException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 997c2c01332..b81c3272356 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -28,6 +28,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -37,9 +38,12 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -561,4 +565,21 @@ public static String getDefaultUnit(String resourceType) {
}
return "";
}
+
+ /**
+ * Get all resource types information from known resource types.
+ * @return List of ResourceTypeInfo
+ */
+ public static List getResourcesTypeInfo() {
+ List array = new ArrayList<>();
+ // Add all resource types
+ Collection resourcesInfo =
+ ResourceUtils.getResourceTypes().values();
+ for (ResourceInformation resourceInfo : resourcesInfo) {
+ array.add(ResourceTypeInfo
+ .newInstance(resourceInfo.getName(), resourceInfo.getUnits(),
+ resourceInfo.getResourceType()));
+ }
+ return array;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 4f5b5bcc647..a0ffe710191 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -72,6 +72,7 @@
import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -863,7 +864,8 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts(
* Get the resource profiles available in the RM.
*
* @return a Map of the resource profile names to their capabilities
- * @throws YarnException if resource profiles are not enabled
+ * @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
@@ -876,9 +878,8 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts(
* Get the details of a specific resource profile from the RM.
*
* @param profile the profile name
- * @return the capabilities of the resource profile
- * @throws YarnException if resource profiles are not enabled or the profile
- * cannot be found
+ * @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other others
*/
@Public
@@ -891,8 +892,7 @@ public abstract Resource getResourceProfile(String profile)
* Get available resource types supported by RM.
*
* @return list of supported resource types with detailed information
- * @throws YarnException if resource profiles are not enabled or the profile
- * cannot be found
+ * @throws YarnException if any issue happens inside YARN
* @throws IOException in case of other others
*/
@Public
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 1e99bc74c0d..ffd4fec0ee0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -528,10 +528,12 @@ private Resource multiplyAndNormalize(Resource r, double by,
long value;
if (stepFactorValue != 0) {
value = roundUp
- ? roundUp((long) Math.ceil(rValue * by), stepFactorValue)
+ ? roundUp((long) Math.ceil((float) (rValue * by)), stepFactorValue)
: roundDown((long) (rValue * by), stepFactorValue);
} else {
- value = roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
+ value = roundUp
+ ? (long) Math.ceil((float) (rValue * by))
+ : (long) (rValue * by);
}
tmp.setValue(value);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java
index 931b1c8b7d5..7ae23e7bb63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java
@@ -82,7 +82,7 @@ public synchronized void addProcessor(
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse resp) throws IOException {
+ RegisterApplicationMasterResponse resp) throws IOException, YarnException {
this.head.registerApplicationMaster(applicationAttemptId, request, resp);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index f657415f827..46dd216ffd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -142,7 +142,6 @@
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ResourceProfilesNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -182,6 +181,7 @@
import org.apache.hadoop.yarn.util.UTCClock;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
@@ -1775,42 +1775,26 @@ public GetAllResourceProfilesResponse getResourceProfiles(
GetAllResourceProfilesRequest request) throws YarnException, IOException {
GetAllResourceProfilesResponse response =
GetAllResourceProfilesResponse.newInstance();
- response.setResourceProfiles(getResourceProfiles());
+ response.setResourceProfiles(resourceProfilesManager.getResourceProfiles());
return response;
}
@Override
public GetResourceProfileResponse getResourceProfile(
GetResourceProfileRequest request) throws YarnException, IOException {
- Map profiles = getResourceProfiles();
- if (!profiles.containsKey(request.getProfileName())) {
- throw new YarnException(
- "Resource profile '" + request.getProfileName() + "' not found");
- }
GetResourceProfileResponse response =
GetResourceProfileResponse.newInstance();
- response.setResource(profiles.get(request.getProfileName()));
+ response.setResource(
+ resourceProfilesManager.getProfile(request.getProfileName()));
return response;
}
- private Map getResourceProfiles() throws YarnException {
- boolean resourceProfilesEnabled = getConfig()
- .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- if (!resourceProfilesEnabled) {
- throw new ResourceProfilesNotEnabledException(
- "Resource profiles are not enabled");
- }
- return resourceProfilesManager.getResourceProfiles();
- }
-
@Override
public GetAllResourceTypeInfoResponse getResourceTypeInfo(
GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
GetAllResourceTypeInfoResponse response =
GetAllResourceTypeInfoResponse.newInstance();
- response.setResourceTypeInfo(
- resourceProfilesManager.getAllResourceTypeInfo());
+ response.setResourceTypeInfo(ResourceUtils.getResourcesTypeInfo());
return response;
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 3f12f31e43f..3a5ab0b9f2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -113,7 +113,8 @@ public void init(ApplicationMasterServiceContext amsContext,
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException {
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException {
RMApp app = getRmContext().getRMApps().get(
applicationAttemptId.getApplicationId());
@@ -173,10 +174,11 @@ public void registerApplicationMaster(
response.setSchedulerResourceTypes(getScheduler()
.getSchedulingResourceTypes());
- if (getRmContext().getYarnConfiguration().getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) {
- response
- .setResourceProfiles(resourceProfilesManager.getResourceProfiles());
+ if (getRmContext().getYarnConfiguration().getBoolean(
+ YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) {
+ response.setResourceProfiles(
+ resourceProfilesManager.getResourceProfiles());
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 3c278de19ab..daf3160747c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -127,7 +127,8 @@ public void init(ApplicationMasterServiceContext amsContext,
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException {
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException {
SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler)
getScheduler()).getApplicationAttempt(applicationAttemptId);
if (appAttempt.getOpportunisticContainerContext() == null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index d3e834578d9..2aae3a5c55e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -628,20 +628,6 @@ public static int getApplicableNodeCountForAM(RMContext rmContext,
}
}
- public static void convertProfileToResourceCapability(
- List asks, Configuration conf,
- ResourceProfilesManager resourceProfilesManager) throws YarnException {
- boolean profilesEnabled =
- conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- if (!profilesEnabled) {
- return;
- }
- for (ResourceRequest req : asks) {
- convertProfileToResourceCapability(req, conf, resourceProfilesManager);
- }
- }
-
public static void convertProfileToResourceCapability(ResourceRequest ask,
Configuration conf, ResourceProfilesManager resourceProfilesManager)
throws YarnException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java
index 896312178ee..700f0ef3aa5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java
@@ -18,22 +18,18 @@
package org.apache.hadoop.yarn.server.resourcemanager.resource;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import java.io.IOException;
-import java.util.List;
import java.util.Map;
/**
* Interface for the resource profiles manager. Provides an interface to get
* the list of available profiles and some helper functions.
*/
-@Public
-@Unstable
public interface ResourceProfilesManager {
/**
@@ -47,14 +43,19 @@
* Get the resource capability associated with given profile name.
* @param profile name of resource profile
* @return resource capability for given profile
+ *
+ * @throws YarnException when any invalid profile name or feature is disabled
*/
- Resource getProfile(String profile);
+ Resource getProfile(String profile) throws YarnException;
/**
* Get all supported resource profiles.
* @return a map of resource objects associated with each profile
+ *
+ * @throws YARNFeatureNotEnabledException when feature is disabled
*/
- Map getResourceProfiles();
+ Map getResourceProfiles() throws
+ YARNFeatureNotEnabledException;
/**
* Reload profiles based on updated configuration.
@@ -65,24 +66,21 @@
/**
* Get default supported resource profile.
* @return resource object which is default
+ * @throws YarnException when any invalid profile name or feature is disabled
*/
- Resource getDefaultProfile();
+ Resource getDefaultProfile() throws YarnException;
/**
* Get minimum supported resource profile.
* @return resource object which is minimum
+ * @throws YarnException when any invalid profile name or feature is disabled
*/
- Resource getMinimumProfile();
+ Resource getMinimumProfile() throws YarnException;
/**
* Get maximum supported resource profile.
* @return resource object which is maximum
+ * @throws YarnException when any invalid profile name or feature is disabled
*/
- Resource getMaximumProfile();
-
- /**
- * List of ResourceTypeInfo objects which carry all resources supported by RM.
- * @return list of ResourceTypeInfo objects
- */
- List getAllResourceTypeInfo();
+ Resource getMaximumProfile() throws YarnException;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
index ff4e3e4ec30..e213a97d711 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
@@ -26,6 +26,8 @@
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.codehaus.jackson.map.ObjectMapper;
@@ -52,6 +54,7 @@
private List resourceTypeInfo =
new ArrayList();
private Configuration conf;
+ private boolean profileEnabled = false;
private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
private static final String VCORES = ResourceInformation.VCORES.getName();
@@ -65,6 +68,11 @@
private static final String[] MANDATORY_PROFILES =
{ DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE };
+ private static final String FEATURE_NOT_ENABLED_MSG =
+ "Resource profile is not enabled, please "
+ + "enable resource profile feature before using its functions."
+ + " (by setting " + YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED
+ + " to true)";
public ResourceProfilesManagerImpl() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@@ -98,10 +106,10 @@ private void loadResourceTypes() {
}
private void loadProfiles() throws IOException {
- boolean profilesEnabled =
+ profileEnabled =
conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- if (!profilesEnabled) {
+ if (!profileEnabled) {
return;
}
String sourceFile =
@@ -131,14 +139,16 @@ private void loadProfiles() throws IOException {
if (entry.getValue() instanceof Map) {
Map profileInfo = (Map) entry.getValue();
// ensure memory and vcores are specified
- if (!profileInfo.containsKey(MEMORY) || !profileInfo.containsKey(VCORES)) {
+ if (!profileInfo.containsKey(MEMORY)
+ || !profileInfo.containsKey(VCORES)) {
throw new IOException(
"Illegal resource profile definition; profile '" + profileName
+ "' must contain '" + MEMORY + "' and '" + VCORES + "'");
}
Resource resource = parseResource(profileInfo);
profiles.put(profileName, resource);
- LOG.info("Added profile '" + profileName + "' with resources " + resource);
+ LOG.info(
+ "Added profile '" + profileName + "' with resources: " + resource);
}
}
// check to make sure mandatory profiles are present
@@ -149,7 +159,7 @@ private void loadProfiles() throws IOException {
+ Arrays.toString(MANDATORY_PROFILES) + " must be present");
}
}
- LOG.info("Loaded profiles " + profiles.keySet());
+ LOG.info("Loaded profiles: " + profiles.keySet());
}
private Resource parseResource(Map profileInfo) throws IOException {
@@ -182,13 +192,33 @@ private Resource parseResource(Map profileInfo) throws IOException {
return resource;
}
+ private void checkAndThrowExceptionWhenFeatureDisabled()
+ throws YARNFeatureNotEnabledException {
+ if (!profileEnabled) {
+ throw new YARNFeatureNotEnabledException(FEATURE_NOT_ENABLED_MSG);
+ }
+ }
+
@Override
- public Resource getProfile(String profile) {
- return Resources.clone(profiles.get(profile));
+ public Resource getProfile(String profile) throws YarnException{
+ checkAndThrowExceptionWhenFeatureDisabled();
+
+ if (profile == null) {
+ throw new YarnException("Profile name cannot be null");
+ }
+
+ Resource profileRes = profiles.get(profile);
+ if (profileRes == null) {
+ throw new YarnException(
+ "Resource profile '" + profile + "' not found");
+ }
+ return Resources.clone(profileRes);
}
@Override
- public Map getResourceProfiles() {
+ public Map getResourceProfiles()
+ throws YARNFeatureNotEnabledException {
+ checkAndThrowExceptionWhenFeatureDisabled();
return Collections.unmodifiableMap(profiles);
}
@@ -200,17 +230,17 @@ public void reloadProfiles() throws IOException {
}
@Override
- public Resource getDefaultProfile() {
+ public Resource getDefaultProfile() throws YarnException {
return getProfile(DEFAULT_PROFILE);
}
@Override
- public Resource getMinimumProfile() {
+ public Resource getMinimumProfile() throws YarnException {
return getProfile(MINIMUM_PROFILE);
}
@Override
- public Resource getMaximumProfile() {
+ public Resource getMaximumProfile() throws YarnException {
return getProfile(MAXIMUM_PROFILE);
}
@@ -220,13 +250,4 @@ private ResourceInformation fromString(String name, String value) {
Long.valueOf(value.substring(0, value.length() - units.length()));
return ResourceInformation.newInstance(name, units, resourceValue);
}
-
- public List getAllResourceTypeInfo() {
- try {
- readLock.lock();
- return Collections.unmodifiableList(resourceTypeInfo);
- } finally {
- readLock.unlock();
- }
- }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 99b798d8ff9..6fe618df853 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -58,6 +58,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
@@ -1279,7 +1280,13 @@ public Resource getMinimumAllocation() {
if (!profilesEnabled) {
ret = ResourceUtils.getResourceTypesMinimumAllocation();
} else {
- ret = rmContext.getResourceProfilesManager().getMinimumProfile();
+ try {
+ ret = rmContext.getResourceProfilesManager().getMinimumProfile();
+ } catch (YarnException e) {
+ LOG.error(
+ "Exception while getting minimum profile from profile manager:", e);
+ throw new YarnRuntimeException(e);
+ }
}
LOG.info("Minimum allocation = " + ret);
return ret;
@@ -1301,7 +1308,14 @@ public Resource getMaximumAllocation() {
if (!profilesEnabled) {
ret = ResourceUtils.getResourceTypesMaximumAllocation();
} else {
- ret = rmContext.getResourceProfilesManager().getMaximumProfile();
+ try {
+ ret = rmContext.getResourceProfilesManager().getMaximumProfile();
+ } catch (YarnException e) {
+ LOG.error(
+ "Exception while getting maximum profile from ResourceProfileManager:",
+ e);
+ throw new YarnRuntimeException(e);
+ }
}
LOG.info("Maximum allocation = " + ret);
return ret;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 4af1e320bbe..2815564368c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -103,9 +103,11 @@ public void init(ApplicationMasterServiceContext amsContext,
}
@Override
- public void registerApplicationMaster(ApplicationAttemptId
- applicationAttemptId, RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException {
+ public void registerApplicationMaster(
+ ApplicationAttemptId applicationAttemptId,
+ RegisterApplicationMasterRequest request,
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException {
nextProcessor.registerApplicationMaster(
applicationAttemptId, request, response);
}
@@ -145,7 +147,8 @@ public void init(ApplicationMasterServiceContext amsContext,
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException {
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException {
beforeRegCount.incrementAndGet();
nextProcessor.registerApplicationMaster(applicationAttemptId,
request, response);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
index c542ed8853f..b53e59ce86e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.Assert;
import org.junit.Test;
@@ -36,8 +37,13 @@ public void testProfilesEnabled() throws Exception {
Configuration conf = new Configuration();
// be default resource profiles should not be enabled
manager.init(conf);
- Map profiles = manager.getResourceProfiles();
- Assert.assertTrue(profiles.isEmpty());
+ try {
+ manager.getResourceProfiles();
+ Assert
+ .fail("Exception should be thrown as resource profile is not enabled"
+ + " and getResourceProfiles is invoked.");
+ } catch (YarnException ie) {
+ }
conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
try {
manager.init(conf);