diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java index b7d925a6592..8e76a11dc27 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java @@ -52,11 +52,13 @@ void init(ApplicationMasterServiceContext amsContext, * @param request Register Request. * @param response Register Response. * @throws IOException IOException. + * @throws YarnException in critical situation where invalid + * profiles/resources are added. */ - void registerApplicationMaster( - ApplicationAttemptId applicationAttemptId, + void registerApplicationMaster(ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException; + RegisterApplicationMasterResponse response) + throws IOException, YarnException; /** * Allocate call. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index c18ec447114..1f0a360b5b2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException; /** *

The protocol between clients and the ResourceManager @@ -598,8 +599,7 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( *

* @param request request to get the details of a resource profile * @return Response containing the details for a particular resource profile - * @throws YarnException if resource profiles are not enabled on the RM or - * the profile cannot be found + * @throws YarnException if any error happens inside YARN * @throws IOException in case of other errors */ @Public diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java new file mode 100644 index 00000000000..62340fea363 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.exceptions; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * This exception is thrown when a feature is being used which is not enabled + * yet. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class YARNFeatureNotEnabledException extends YarnException { + private static final long serialVersionUID = 898023752676L; + + public YARNFeatureNotEnabledException(Throwable cause) { + super(cause); + } + + public YARNFeatureNotEnabledException(String message) { + super(message); + } + + public YARNFeatureNotEnabledException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 997c2c01332..b81c3272356 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -37,9 +38,12 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -561,4 +565,21 @@ public static String getDefaultUnit(String resourceType) { } return ""; } + + /** + * Get all resource types information from known resource types. + * @return List of ResourceTypeInfo + */ + public static List getResourcesTypeInfo() { + List array = new ArrayList<>(); + // Add all resource types + Collection resourcesInfo = + ResourceUtils.getResourceTypes().values(); + for (ResourceInformation resourceInfo : resourcesInfo) { + array.add(ResourceTypeInfo + .newInstance(resourceInfo.getName(), resourceInfo.getUnits(), + resourceInfo.getResourceType())); + } + return array; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 5435bde1bc8..a3505054326 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; +import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; @@ -863,8 +864,7 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( * Get available resource types supported by RM. *

* @return list of supported resource types with detailed information - * @throws YarnException if resource profiles are not enabled or the profile - * cannot be found + * @throws YarnException if any issue happens inside YARN * @throws IOException in case of other others */ @Public diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 1e99bc74c0d..ffd4fec0ee0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -528,10 +528,12 @@ private Resource multiplyAndNormalize(Resource r, double by, long value; if (stepFactorValue != 0) { value = roundUp - ? roundUp((long) Math.ceil(rValue * by), stepFactorValue) + ? roundUp((long) Math.ceil((float) (rValue * by)), stepFactorValue) : roundDown((long) (rValue * by), stepFactorValue); } else { - value = roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by); + value = roundUp + ? (long) Math.ceil((float) (rValue * by)) + : (long) (rValue * by); } tmp.setValue(value); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 9e4a0d6c9f9..952f08696e1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -421,6 +421,7 @@ public static void setup() throws Exception { generateByNewInstance(ApplicationTimeout.class); generateByNewInstance(QueueConfigurations.class); generateByNewInstance(CollectorInfo.class); + generateByNewInstance(ResourceTypeInfo.class); } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java index 931b1c8b7d5..7ae23e7bb63 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java @@ -82,7 +82,7 @@ public synchronized void addProcessor( public void registerApplicationMaster( ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse resp) throws IOException { + RegisterApplicationMasterResponse resp) throws IOException, YarnException { this.head.registerApplicationMaster(applicationAttemptId, request, resp); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index a0ed07b66c0..b2c5682510f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -178,6 +178,7 @@ import org.apache.hadoop.yarn.util.UTCClock; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; @@ -1806,8 +1807,7 @@ public GetAllResourceTypeInfoResponse getResourceTypeInfo( GetAllResourceTypeInfoRequest request) throws YarnException, IOException { GetAllResourceTypeInfoResponse response = GetAllResourceTypeInfoResponse.newInstance(); - response.setResourceTypeInfo( - resourceProfilesManager.getAllResourceTypeInfo()); + response.setResourceTypeInfo(ResourceUtils.getResourcesTypeInfo()); return response; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index d5444b48091..a68bf3166f4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -111,7 +111,8 @@ public void init(ApplicationMasterServiceContext amsContext, public void registerApplicationMaster( ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException { + RegisterApplicationMasterResponse response) + throws IOException, YarnException { RMApp app = getRmContext().getRMApps().get( applicationAttemptId.getApplicationId()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java index 98944af6e92..ce425dfaac4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java @@ -127,7 +127,8 @@ public void init(ApplicationMasterServiceContext amsContext, public void registerApplicationMaster( ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException { + RegisterApplicationMasterResponse response) + throws IOException, YarnException { SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler) getScheduler()).getApplicationAttempt(applicationAttemptId); if (appAttempt.getOpportunisticContainerContext() == null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 6be000eb1e0..211c9f653b1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java index e684f3c1ad3..421ddbc29b6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java @@ -102,9 +102,11 @@ public void init(ApplicationMasterServiceContext amsContext, } @Override - public void registerApplicationMaster(ApplicationAttemptId - applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException { + public void registerApplicationMaster( + ApplicationAttemptId applicationAttemptId, + RegisterApplicationMasterRequest request, + RegisterApplicationMasterResponse response) + throws IOException, YarnException { nextProcessor.registerApplicationMaster( applicationAttemptId, request, response); } @@ -144,7 +146,8 @@ public void init(ApplicationMasterServiceContext amsContext, public void registerApplicationMaster( ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException { + RegisterApplicationMasterResponse response) + throws IOException, YarnException { beforeRegCount.incrementAndGet(); nextProcessor.registerApplicationMaster(applicationAttemptId, request, response);