diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml index 691b5455994..cbb0c68d79d 100644 --- a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml +++ b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml @@ -84,7 +84,7 @@ but enforcer still sees it. --> org.apache.hadoop:hadoop-annotations - + org.apache.htrace:htrace-core4 org.slf4j:slf4j-api diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 00f2d254035..3de5091aa05 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -571,7 +571,7 @@ org.apache.hadoop:hadoop-yarn-server-timelineservice log4j:log4j - + @@ -664,6 +664,27 @@ krb5.conf + + org.eclipse.persistence:eclipselink + + **/*.html + + + + org.eclipse.persistence:javax.persistence + + **/*.html + + + + org.eclipse.persistence:commonj.sdo + + **/*.html + about_files/** + xsd/** + plugin.properties + + @@ -782,6 +803,35 @@ **/pom.xml + + + javax/persistence/ + ${shaded.dependency.prefix}.javax.persistence. + + **/pom.xml + + + + javax/validation/ + ${shaded.dependency.prefix}.javax.validation. + + **/pom.xml + + + + commonj/ + ${shaded.dependency.prefix}.commonj. + + **/pom.xml + + + + javax/json/ + ${shaded.dependency.prefix}.javax.json. + + **/pom.xml + + jersey/ ${shaded.dependency.prefix}.jersey. diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3c49182b2f0..46c4f0ec449 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -146,6 +146,7 @@ 1.5.4 1.16 + 2.6.0 @@ -1357,6 +1358,11 @@ 3.8.0 test + + org.eclipse.persistence + eclipselink + ${eclipselink.version} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index a4d7afc1566..b8bb9eb5915 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -250,6 +250,15 @@ jersey-test-framework-grizzly2 test + + + + org.eclipse.persistence + eclipselink + @@ -357,6 +366,7 @@ src/test/resources/profiles/illegal-profiles-1.json src/test/resources/profiles/illegal-profiles-2.json src/test/resources/profiles/illegal-profiles-3.json + src/main/resources/META-INF/services/javax.ws.rs.ext.MessageBodyWriter diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index a0317f62cc0..d228fbb1dde 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import com.google.common.annotations.VisibleForTesting; +import com.sun.jersey.spi.service.ServiceFinder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.curator.framework.AuthInfo; @@ -100,6 +101,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper + .CustomServiceIteratorProvider; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; @@ -268,7 +271,7 @@ protected void serviceInit(Configuration conf) throws Exception { loadConfigurationXml(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); validateConfigs(this.conf); - + // Set HA configuration should be done before login this.rmContext.setHAEnabled(HAUtil.isHAEnabled(this.conf)); if (this.rmContext.isHAEnabled()) { @@ -312,7 +315,7 @@ protected void serviceInit(Configuration conf) throws Exception { } rmContext.setYarnConfiguration(conf); - + createAndInitActiveServices(false); webAppAddress = WebAppUtils.getWebAppBindURL(this.conf, @@ -485,12 +488,12 @@ private NMLivelinessMonitor createNMLivelinessMonitor() { protected AMLivelinessMonitor createAMLivelinessMonitor() { return new AMLivelinessMonitor(this.rmDispatcher); } - + protected RMNodeLabelsManager createNodeLabelManager() throws InstantiationException, IllegalAccessException { return new RMNodeLabelsManager(); } - + protected DelegationTokenRenewer createDelegationTokenRenewer() { return new DelegationTokenRenewer(); } @@ -599,7 +602,7 @@ protected void serviceInit(Configuration configuration) throws Exception { AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor(); addService(amFinishingMonitor); rmContext.setAMFinishingMonitor(amFinishingMonitor); - + RMAppLifetimeMonitor rmAppLifetimeMonitor = createRMAppLifetimeMonitor(); addService(rmAppLifetimeMonitor); rmContext.setRMAppLifetimeMonitor(rmAppLifetimeMonitor); @@ -1041,7 +1044,7 @@ protected void startWepApp() { RMWebAppUtil.setupSecurityAndFilters(conf, getClientRMService().rmDTSecretManager); - Builder builder = + Builder builder = WebApps .$for("cluster", ApplicationMasterService.class, masterService, "ws") @@ -1109,6 +1112,14 @@ protected void startWepApp() { serviceConfig.put("PackageName", apiPackages); serviceConfig.put("PathSpec", "/app/*"); } + + /** + * Introduced in YARN-7451. + * See comments here: {@link CustomServiceIteratorProvider} + */ + ServiceFinder.setIteratorProvider(new CustomServiceIteratorProvider()); + + webApp = builder.start(new RMWebApp(this), uiWebAppContext, serviceConfig); } @@ -1227,7 +1238,7 @@ protected void serviceStart() throws Exception { } super.serviceStart(); } - + protected void doSecureLogin() throws IOException { InetSocketAddress socAddr = getBindAddress(conf); SecurityUtil.login(this.conf, YarnConfiguration.RM_KEYTAB, @@ -1257,7 +1268,7 @@ protected void serviceStop() throws Exception { transitionToStandby(false); rmContext.setHAServiceState(HAServiceState.STOPPING); } - + protected ResourceTrackerService createResourceTrackerService() { return new ResourceTrackerService(this.rmContext, this.nodesListManager, this.nmLivelinessMonitor, @@ -1465,7 +1476,7 @@ private void setSchedulerRecoveryStartAndWaitTime(RMState state, /** * Retrieve RM bind address from configuration - * + * * @param conf * @return InetSocketAddress */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/Application.java new file mode 100644 index 00000000000..70063ad80c7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/Application.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import com.sun.jersey.api.core.DefaultResourceConfig; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.JSONRootElementProviderEclipseLink; + +/** + * + */ +public class Application extends DefaultResourceConfig { + public Application() { + super(JSONRootElementProviderEclipseLink.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 2f50a24eac3..06b05fb4642 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; import com.google.inject.Singleton; import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONJAXBContext; @@ -27,54 +29,128 @@ import javax.ws.rs.ext.ContextResolver; import javax.ws.rs.ext.Provider; import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*; import org.apache.hadoop.yarn.webapp.RemoteExceptionData; +import org.eclipse.persistence.jaxb.JAXBContextFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Singleton @Provider public class JAXBContextResolver implements ContextResolver { + private static final Logger LOG = + LoggerFactory.getLogger(JAXBContextResolver.class); + static final JSONConfiguration JSON_CONFIG_DEFAULT = + JSONConfiguration.natural().rootUnwrapping(false).build(); + + private static final JSONConfiguration JSON_CONFIG_UNWRAPPED = + JSONConfiguration.natural().rootUnwrapping(true).build(); private final Map typesContextMap; + /** + * You have to specify all the dao classes here. + */ + private static final Class[] NORMAL_TYPES = new Class[] { + AppAttemptInfo.class, AppAttemptsInfo.class, ClusterInfo.class, + CapacitySchedulerQueueInfo.class, FifoSchedulerInfo.class, NodeInfo.class, + UserMetricsInfo.class, CapacitySchedulerInfo.class, + ClusterMetricsInfo.class, SchedulerInfo.class, NodesInfo.class, + RemoteExceptionData.class, CapacitySchedulerQueueInfoList.class, + ResourceInfo.class, UsersInfo.class, UserInfo.class, + ApplicationStatisticsInfo.class, StatisticsItemInfo.class, + CapacitySchedulerHealthInfo.class, FairSchedulerQueueInfoList.class, + AppTimeoutsInfo.class, AppTimeoutInfo.class }; + + /** + * These dao classes need root unwrapping. + */ + private static final Class[] ROOT_UNWRAPPED_TYPES = new Class[] { + NewApplication.class, ApplicationSubmissionContextInfo.class, + ContainerLaunchContextInfo.class, LocalResourceInfo.class, + DelegationToken.class, AppQueue.class, AppPriority.class }; + + /** + * Introduced in YARN-7451. + * These classes need special handling because of custom resource types + * should be serialized as a flattened-map along with vCores and memory, + * see {@see ResourceInfoWithCustomResourceTypes} for reference. + * The representation classes listed here contains + * {@link ResourceInfoWithCustomResourceTypes} either directly or indirectly. + */ + public static final List ECLIPSELINK_SERIALIZED_TYPES = + ImmutableList.of(SchedulerTypeInfo.class, AppsInfo.class, + AppInfo.class, ResourceInfoWithCustomResourceTypes.class); + public JAXBContextResolver() throws Exception { + this.typesContextMap = createTypesContextMap(); + } + + private Map createTypesContextMap() throws Exception { + Map jaxbContexts = Maps.newHashMap(); + jaxbContexts.putAll(getJAXBContextsForNormalClasses()); + jaxbContexts.putAll(getJAXBContextsForUnwrappedClasses()); + jaxbContexts.putAll(getJAXBContextForEclipseLinkClasses()); - JAXBContext context; - JAXBContext unWrappedRootContext; - - // you have to specify all the dao classes here - final Class[] cTypes = - { AppInfo.class, AppAttemptInfo.class, AppAttemptsInfo.class, - ClusterInfo.class, CapacitySchedulerQueueInfo.class, - FifoSchedulerInfo.class, SchedulerTypeInfo.class, NodeInfo.class, - UserMetricsInfo.class, CapacitySchedulerInfo.class, - ClusterMetricsInfo.class, SchedulerInfo.class, AppsInfo.class, - NodesInfo.class, RemoteExceptionData.class, - CapacitySchedulerQueueInfoList.class, ResourceInfo.class, - UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, - StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, - FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, - AppTimeoutInfo.class }; - // these dao classes need root unwrapping - final Class[] rootUnwrappedTypes = - { NewApplication.class, ApplicationSubmissionContextInfo.class, - ContainerLaunchContextInfo.class, LocalResourceInfo.class, - DelegationToken.class, AppQueue.class, AppPriority.class }; - - this.typesContextMap = new HashMap(); - context = - new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(false) - .build(), cTypes); - unWrappedRootContext = - new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(true) - .build(), rootUnwrappedTypes); - for (Class type : cTypes) { - typesContextMap.put(type, context); + return jaxbContexts; + } + + private Map getJAXBContextsForNormalClasses() + throws JAXBException { + LOG.debug( + "Registering classes with json config without root unwrapping: {}", + Arrays.toString(NORMAL_TYPES)); + Map result = Maps.newHashMap(); + + JAXBContext context = + new JSONJAXBContext(JSON_CONFIG_DEFAULT, NORMAL_TYPES); + for (Class type : NORMAL_TYPES) { + result.put(type, context); } - for (Class type : rootUnwrappedTypes) { - typesContextMap.put(type, unWrappedRootContext); + + return result; + } + + private Map getJAXBContextsForUnwrappedClasses() + throws JAXBException { + LOG.debug("Registering classes with json config with root unwrapping: {}", + Arrays.toString(ROOT_UNWRAPPED_TYPES)); + Map result = Maps.newHashMap(); + + JAXBContext context = + new JSONJAXBContext(JSON_CONFIG_UNWRAPPED, ROOT_UNWRAPPED_TYPES); + for (Class type : ROOT_UNWRAPPED_TYPES) { + result.put(type, context); + } + + return result; + } + + /** + * Sse Eclipselink JAXBProvider for the specified classes, see + * ResourceInfoWithCustomResourceTypes.customResources as an example, it uses + * EclipseLink JAXBProvider's @XMlPath annotation. + */ + private Map getJAXBContextForEclipseLinkClasses() + throws JAXBException { + final Class[] eclipseLinkTypesArray = ECLIPSELINK_SERIALIZED_TYPES + .toArray(new Class[ECLIPSELINK_SERIALIZED_TYPES.size()]); + + LOG.debug("Registering classes with json config EclipseLink marshaller: {}", + Arrays.toString(eclipseLinkTypesArray)); + Map result = Maps.newHashMap(); + + JAXBContext context = JAXBContextFactory.createContext( + eclipseLinkTypesArray, Collections. emptyMap()); + + for (Class type : eclipseLinkTypesArray) { + result.put(type, context); } + + return result; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 0711b457a5e..34e27dfb1af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -21,11 +21,8 @@ import java.util.List; import java.util.Map; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.*; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -46,11 +43,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields; import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields.DeSelectType; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.MapAdapter; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; +import org.eclipse.persistence.oxm.annotations.XmlPath; @XmlRootElement(name = "app") @XmlAccessorType(XmlAccessType.FIELD) @@ -101,6 +101,11 @@ private long vcoreSeconds; protected float queueUsagePercentage; protected float clusterUsagePercentage; + + // EclipseLink JAXB provider would not include this field if empty without + // these annotations + @XmlJavaTypeAdapter(MapAdapter.class) + @XmlPath("resourceSecondsMap") protected Map resourceSecondsMap; // preemption info fields @@ -110,12 +115,15 @@ private int numAMContainerPreempted; private long preemptedMemorySeconds; private long preemptedVcoreSeconds; + + // EclipseLink JAXB provider would not include this field if empty without + // these annotations + @XmlJavaTypeAdapter(MapAdapter.class) + @XmlPath("preemptedResourceSecondsMap") protected Map preemptedResourceSecondsMap; - // list of resource requests - @XmlElement(name = "resourceRequests") private List resourceRequests = - new ArrayList(); + new ArrayList<>(); protected LogAggregationStatus logAggregationStatus; protected boolean unmanagedApplication; @@ -470,7 +478,7 @@ public long getPreemptedVCores() { public int getNumNonAMContainersPreempted() { return numNonAMContainerPreempted; } - + public int getNumAMContainersPreempted() { return numAMContainerPreempted; } @@ -499,6 +507,11 @@ public long getPreemptedVcoreSeconds() { return preemptedResourceSecondsMap; } + public void setPreemptedResourceSecondsMap( + Map preemptedResourceSecondsMap) { + this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; + } + public List getResourceRequests() { return this.resourceRequests; } @@ -615,4 +628,8 @@ public void setState(YarnApplicationState state) { public void setName(String name) { this.name = name; } + + public void setResourceSecondsMap(Map resourceSecondsMap) { + this.resourceSecondsMap = resourceSecondsMap; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java index 913513c52ae..c7adfa7e913 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -38,9 +38,9 @@ @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @XmlSeeAlso({FairSchedulerLeafQueueInfo.class}) -public class FairSchedulerQueueInfo { +public class FairSchedulerQueueInfo { private int maxApps; - + @XmlTransient private float fractionMemUsed; @XmlTransient @@ -49,14 +49,14 @@ private float fractionMemFairShare; @XmlTransient private float fractionMemMaxShare; - + private ResourceInfo minResources; - private ResourceInfo maxResources; - private ResourceInfo usedResources; + private ResourceInfoWithCustomResourceTypes maxResources; + private ResourceInfoWithCustomResourceTypes usedResources; private ResourceInfo amUsedResources; private ResourceInfo amMaxResources; - private ResourceInfo demandResources; - private ResourceInfo steadyFairResources; + private ResourceInfoWithCustomResourceTypes demandResources; + private ResourceInfoWithCustomResourceTypes steadyFairResources; private ResourceInfo fairResources; private ResourceInfo clusterResources; private ResourceInfo reservedResources; @@ -73,40 +73,45 @@ public FairSchedulerQueueInfo() { } - + public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); - + queueName = queue.getName(); schedulingPolicy = queue.getPolicy().getName(); - + clusterResources = new ResourceInfo(scheduler.getClusterResource()); - + amUsedResources = new ResourceInfo(Resource.newInstance( queue.getMetrics().getAMResourceUsageMB(), queue.getMetrics().getAMResourceUsageVCores())); amMaxResources = new ResourceInfo(Resource.newInstance( queue.getMetrics().getMaxAMShareMB(), queue.getMetrics().getMaxAMShareVCores())); - usedResources = new ResourceInfo(queue.getResourceUsage()); - demandResources = new ResourceInfo(queue.getDemand()); + usedResources = new ResourceInfoWithCustomResourceTypes( + queue.getResourceUsage()); + demandResources = new ResourceInfoWithCustomResourceTypes( + queue.getDemand()); fractionMemUsed = (float)usedResources.getMemorySize() / clusterResources.getMemorySize(); - steadyFairResources = new ResourceInfo(queue.getSteadyFairShare()); + steadyFairResources = new ResourceInfoWithCustomResourceTypes( + queue.getSteadyFairShare()); fairResources = new ResourceInfo(queue.getFairShare()); minResources = new ResourceInfo(queue.getMinShare()); - maxResources = new ResourceInfo( + maxResources = new ResourceInfoWithCustomResourceTypes( Resources.componentwiseMin(queue.getMaxShare(), scheduler.getClusterResource())); reservedResources = new ResourceInfo(queue.getReservedResource()); fractionMemSteadyFairShare = - (float)steadyFairResources.getMemorySize() / clusterResources.getMemorySize(); + (float)steadyFairResources.getMemorySize() + / clusterResources.getMemorySize(); fractionMemFairShare = (float) fairResources.getMemorySize() / clusterResources.getMemorySize(); - fractionMemMaxShare = (float)maxResources.getMemorySize() / clusterResources.getMemorySize(); - + fractionMemMaxShare = (float)maxResources.getMemorySize() / + clusterResources.getMemorySize(); + maxApps = queue.getMaxRunningApps(); allocatedContainers = queue.getMetrics().getAllocatedContainers(); @@ -130,7 +135,8 @@ public long getReservedContainers() { } protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue, - FairScheduler scheduler) { + FairScheduler scheduler) + { // Return null to omit 'childQueues' field from the return value of // REST API if it is empty. We omit the field to keep the consistency // with CapacitySchedulerQueueInfo, which omits 'queues' field if empty. @@ -150,7 +156,7 @@ protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue, } return list; } - + /** * Returns the steady fair share as a fraction of the entire cluster capacity. */ @@ -168,7 +174,7 @@ public float getFairShareMemoryFraction() { /** * Returns the steady fair share of this queue in megabytes. */ - public ResourceInfo getSteadyFairShare() { + public ResourceInfoWithCustomResourceTypes getSteadyFairShare() { return steadyFairResources; } @@ -182,11 +188,11 @@ public ResourceInfo getFairShare() { public ResourceInfo getMinResources() { return minResources; } - - public ResourceInfo getMaxResources() { + + public ResourceInfoWithCustomResourceTypes getMaxResources() { return maxResources; } - + public ResourceInfo getReservedResources() { return reservedResources; } @@ -194,12 +200,12 @@ public ResourceInfo getReservedResources() { public int getMaxApplications() { return maxApps; } - + public String getQueueName() { return queueName; } - - public ResourceInfo getUsedResources() { + + public ResourceInfoWithCustomResourceTypes getUsedResources() { return usedResources; } @@ -220,26 +226,26 @@ public ResourceInfo getAMMaxResources() { /** * @return the demand resource of this queue. */ - public ResourceInfo getDemandResources() { + public ResourceInfoWithCustomResourceTypes getDemandResources() { return demandResources; } /** - * Returns the memory used by this queue as a fraction of the entire + * Returns the memory used by this queue as a fraction of the entire * cluster capacity. */ public float getUsedMemoryFraction() { return fractionMemUsed; } - + /** - * Returns the capacity of this queue as a fraction of the entire cluster + * Returns the capacity of this queue as a fraction of the entire cluster * capacity. */ public float getMaxResourcesFraction() { return fractionMemMaxShare; } - + /** * Returns the name of the scheduling policy used by this queue. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfoWithCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfoWithCustomResourceTypes.java new file mode 100644 index 00000000000..3c1eef1687f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfoWithCustomResourceTypes.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.MapAdapter; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.eclipse.persistence.oxm.annotations.XmlPath; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import java.util.Map; +import java.util.Set; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.NONE) +public class ResourceInfoWithCustomResourceTypes { + private static final Set DEFAULT_RESOURCE_NAMES = + Sets.newHashSet("memory", "vcores"); + + @XmlElement + private long memory; + @XmlElement + private int vCores; + + private Resource resources; + + /** + * XmlPath produces a flattened structure, like the keys / values of map were + * fields of this object. Example: + * "maxResources": + * {"memory":0,"vCores":0,"customResource1":11,"customResource2":22} + */ + @XmlJavaTypeAdapter(MapAdapter.class) + @XmlPath(".") + private Map customResources = Maps.newHashMap(); + + public ResourceInfoWithCustomResourceTypes() { + } + + ResourceInfoWithCustomResourceTypes(final Resource res) { + memory = res.getMemorySize(); + vCores = res.getVirtualCores(); + resources = Resources.clone(res); + customResources = createCustomResources(res); + } + + /** + * + * @param res + * @return + */ + private static Map createCustomResources(final Resource res) { + final ResourceInformation[] resourceInformations = res.getResources(); + + final Map result = Maps.newHashMap(); + for (ResourceInformation ri : resourceInformations) { + final String name = ri.getName(); + + if (!isDefaultResource(name)) { + result.put(name, ri.getValue()); + } + } + + return result; + } + + private static boolean isDefaultResource(final String name) { + final String nameLowerCase = name.toLowerCase(); + + for (String defaultResourceName : DEFAULT_RESOURCE_NAMES) { + if (nameLowerCase.contains(defaultResourceName)) { + return true; + } + } + return false; + } + + public long getMemorySize() { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + return resources.getMemorySize(); + } + + public int getvCores() { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + return resources.getVirtualCores(); + } + + public void setvCores(int vCores) { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + this.vCores = vCores; + resources.setVirtualCores(vCores); + } + + + public long getMemory() { + return memory; + } + + public void setMemory(int memory) { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + this.memory = memory; + resources.setMemorySize(memory); + } + + + @Override + public String toString() { + return resources.toString(); + } + + public Resource getResource() { + return Resource.newInstance(resources); + } + + + public Map getCustomResources() { + return customResources; + } + + public void setCustomResources(Map customResources) { + this.customResources = customResources; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java index 030af45cced..932bfff98bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java @@ -39,7 +39,7 @@ @XmlElement(name = "resourceName") private String resourceName; @XmlElement(name = "capability") - private ResourceInfo capability; + private ResourceInfoWithCustomResourceTypes capability; @XmlElement(name = "numContainers") private int numContainers; @XmlElement(name = "relaxLocality") @@ -61,7 +61,8 @@ public ResourceRequestInfo() { public ResourceRequestInfo(ResourceRequest request) { priority = request.getPriority().getPriority(); resourceName = request.getResourceName(); - capability = new ResourceInfo(request.getCapability()); + capability = new ResourceInfoWithCustomResourceTypes( + request.getCapability()); numContainers = request.getNumContainers(); relaxLocality = request.getRelaxLocality(); nodeLabelExpression = request.getNodeLabelExpression(); @@ -87,11 +88,11 @@ public void setResourceName(String resourceName) { this.resourceName = resourceName; } - public ResourceInfo getCapability() { + public ResourceInfoWithCustomResourceTypes getCapability() { return capability; } - public void setCapability(ResourceInfo capability) { + public void setCapability(ResourceInfoWithCustomResourceTypes capability) { this.capability = capability; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java index 81491b14ce1..3030faffa01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java @@ -23,6 +23,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; +import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -31,6 +32,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +/** + * + */ @XmlRootElement @XmlSeeAlso({ CapacitySchedulerInfo.class, FairSchedulerInfo.class, FifoSchedulerInfo.class }) @@ -41,8 +45,9 @@ protected EnumSet schedulingResourceTypes; protected int maximumClusterPriority; + // JAXB needs this public SchedulerInfo() { - } // JAXB needs this + } public SchedulerInfo(final ResourceManager rm) { ResourceScheduler rs = rm.getResourceScheduler(); @@ -74,9 +79,19 @@ public ResourceInfo getMaxAllocation() { } public String getSchedulerResourceTypes() { - return Arrays.toString(minAllocResource.getResource().getResources()); + if (minAllocResource != null) { + return Arrays.toString(minAllocResource.getResource().getResources()); + } + return null; } + /** + * Should apply @{@link XmlTransient} as when the EclipseLink serializer was + * introduced in YARN-7451, maximumClusterPriority appeared in the response, + * but according to tests, it should not appear in the responses. + * @return The maximum cluster priority + */ + @XmlTransient public int getMaxClusterLevelAppPriority() { return this.maximumClusterPriority; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CustomServiceIteratorProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CustomServiceIteratorProvider.java new file mode 100644 index 00000000000..19993002a9d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CustomServiceIteratorProvider.java @@ -0,0 +1,266 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; + +import com.sun.jersey.spi.service.ServiceConfigurationError; +import com.sun.jersey.spi.service.ServiceFinder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.ext.MessageBodyWriter; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.util.*; +import java.util.regex.Pattern; + +/** + * Despite Jersey's documentation clearly declares that a user created + * {@link MessageBodyWriter} like the {@link JSONRootElementProviderEclipseLink} + * would be chosen in favor of the Providers defined by Jersey while serializing + * response objects, on the contrary, it does not work like that.
+ * Apparently, the iteration order defined by the return value of the method + * {@link CustomServiceIteratorProvider#createClassIterator(Class, String, ClassLoader, boolean)} + * matters in terms of choosing the preferred {@link MessageBodyWriter}.
+ *
+ * See the method + * {@link CustomServiceIteratorProvider#reorderURLsIfNeeded(Class, List)} for + * details. + * + *

+ * Based on the above findings, this class creates a custom iterator provider + * that is heavily based on Jersey's default implementation, see: + * {@link com.sun.jersey.spi.service.ServiceFinder.DefaultServiceIteratorProvider} + * + * Both the {@link com.sun.jersey.spi.service.ServiceFinder.LazyObjectIterator} + * and {@link com.sun.jersey.spi.service.ServiceFinder.LazyClassIterator} are + * created with reflection, since Jersey's implementation is too rigid, they + * used private classes and methods nearly everywhere in + * {@link com.sun.jersey.spi.service.ServiceFinder}, + * making it impossible to extend in any way. + * All calls are delegated + * with reflection to the default provider of Jersey, while ensuring that custom + * implementations of {@link MessageBodyWriter} + * are always returned first with the iterators. + * The only catch is right after the iterators are created with reflection, + * the {@link CustomServiceIteratorProvider#setConfigs(Object)} method is called, + * ensuring that the internal "configs" field is set + * in {@link com.sun.jersey.spi.service.ServiceFinder.AbstractLazyIterator}, + * so the ordering of the classes will be correct and will be preserved. + * + * Please note that the simplest way to solve this situation would have been to + * copy the original {@link com.sun.jersey.spi.service.ServiceFinder} and patch + * the ordering in the copied code, but unfortunately it is not possible to + * include any of the source files of Jersey due to licensing issues, see this + * legal discussion for details: + * https://issues.apache.org/jira/browse/LEGAL-359. + * + */ +public final class CustomServiceIteratorProvider extends + ServiceFinder.ServiceIteratorProvider { + private static final Logger LOG = + LoggerFactory.getLogger(CustomServiceIteratorProvider.class); + + private static final String PREFIX = "META-INF/services/"; + + @Override + public Iterator createIterator(Class service, String serviceName, + ClassLoader loader, boolean ignoreOnClassNotFound) { + return (Iterator) createIteratorInternal(service, serviceName, loader, + ignoreOnClassNotFound, + "com.sun.jersey.spi.service.ServiceFinder$LazyObjectIterator"); + } + + @Override + public Iterator> createClassIterator(Class service, + String serviceName, ClassLoader loader, boolean ignoreOnClassNotFound) { + return (Iterator>) createIteratorInternal(service, serviceName, + loader, ignoreOnClassNotFound, + "com.sun.jersey.spi.service.ServiceFinder$LazyClassIterator"); + } + + private Object createIteratorInternal(Class service, String serviceName, + ClassLoader loader, boolean ignoreOnClassNotFound, + String iteratorClassName) { + try { + Class iteratorClass = Class.forName(iteratorClassName); + Constructor iteratorConstructor = iteratorClass.getDeclaredConstructor( + Class.class, String.class, ClassLoader.class, boolean.class); + iteratorConstructor.setAccessible(true); + Object iterator = iteratorConstructor.newInstance(service, serviceName, + loader, ignoreOnClassNotFound); + + logConfigs(iterator); + setConfigs(iterator); + + return iterator; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static void logConfigs(Object iterator) { + try { + Field serviceNameField = getField(iterator, "serviceName"); + String serviceName = (String) serviceNameField.get(iterator); + + Field loaderField = getField(iterator, "loader"); + ClassLoader loader = (ClassLoader) loaderField.get(iterator); + + final String fullName = PREFIX + serviceName; + + try { + Enumeration resources = invokeGetResources(loader, fullName); + List urls = filterServiceURLsWithVersion(fullName, resources); + logConfig(serviceName, urls); + } catch (InvocationTargetException ite) { + if (ite.getCause() instanceof IOException) { + fail(serviceName, ": " + ite.getCause()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static void setConfigs(Object iterator) { + try { + Field configsField = getField(iterator, "configs"); + if (configsField.get(iterator) == null) { + Field loaderField = getField(iterator, "loader"); + ClassLoader loader = (ClassLoader) loaderField.get(iterator); + + Field serviceNameField = getField(iterator, "serviceName"); + String serviceName = (String) serviceNameField.get(iterator); + + Field serviceField = getField(iterator, "service"); + Class service = (Class) serviceField.get(iterator); + try { + final String fullName = PREFIX + serviceName; + Enumeration resources = invokeGetResources(loader, fullName); + List urls = filterServiceURLsWithVersion(fullName, resources); + reorderURLsIfNeeded(service, urls); + configsField.set(iterator, Collections.enumeration(urls)); + } catch (InvocationTargetException ite) { + if (ite.getCause() instanceof IOException) { + fail(serviceName, ": " + ite.getCause()); + } + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static void logConfig(String serviceName, List urls) { + StringBuilder sb = new StringBuilder(); + for (URL url : urls) { + sb.append(url.toString()).append(", "); + } + + LOG.debug("ServiceFinder config for class: {}, configuration URLs: " + + serviceName, sb.toString()); + } + + /** + * Reorder the provided urls list so that the jar file + * hadoop-yarn-server-resourcemanager.jar is always in the first place in the + * list only if the service is {@link MessageBodyWriter}. + * + * @param service + * @param urls + */ + private static void reorderURLsIfNeeded(Class service, List urls) { + // format of jar file name: + // hadoop-yarn-server-resourcemanager-3.1.0-SNAPSHOT.jar + if (service.equals(MessageBodyWriter.class)) { + int indexOfResource = + getIndexOfResource(urls, "hadoop-yarn-server-resourcemanager"); + if (indexOfResource != -1) { + Collections.swap(urls, 0, indexOfResource); + } + } + } + + private static int getIndexOfResource(List urls, String resourceName) { + for (int i = 0; i < urls.size(); i++) { + URL url = urls.get(i); + if (url != null && url.getPath() != null && url.getPath() + .matches(".*" + Pattern.quote(resourceName) + ".*\\.jar.*")) { + return i; + } + } + return -1; + } + + private static Field getField(Object iterator, String fieldName) { + try { + Field field = iterator.getClass().getSuperclass() + .getDeclaredField(fieldName); + field.setAccessible(true); + return field; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static Enumeration invokeGetResources(ClassLoader loader, + String fullName) + throws InvocationTargetException { + return (Enumeration) invokeStaticMethod( + ServiceFinder.class, "getResources", + new Class[] { ClassLoader.class, String.class }, loader, fullName); + } + + private static List filterServiceURLsWithVersion( + String serviceName, + Enumeration serviceUrls) { + try { + Enumeration enumeration = (Enumeration) invokeStaticMethod( + ServiceFinder.class, + "filterServiceURLsWithVersion", + new Class[] { String.class, Enumeration.class }, serviceName, + serviceUrls); + return Collections.list(enumeration); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } + } + + private static Object invokeStaticMethod(Class clazz, String methodName, + Class[] parameterTypes, Object... args) throws InvocationTargetException { + try { + Method method = clazz.getDeclaredMethod(methodName, parameterTypes); + method.setAccessible(true); + return method.invoke(null, args); + } catch (InvocationTargetException ite) { + throw ite; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static void fail(String serviceName, String msg) + throws com.sun.jersey.spi.service.ServiceConfigurationError { + throw new ServiceConfigurationError(serviceName + ": " + msg); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/JSONRootElementProviderEclipseLink.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/JSONRootElementProviderEclipseLink.java new file mode 100644 index 00000000000..17c5e474488 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/JSONRootElementProviderEclipseLink.java @@ -0,0 +1,266 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; + +import com.google.common.collect.ImmutableSet; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; +import com.sun.jersey.core.header.QualitySourceMediaType; +import com.sun.jersey.core.provider.jaxb.AbstractRootElementProvider; +import com.sun.jersey.core.util.FeaturesAndProperties; +import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; + + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.Consumes; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.ext.Providers; +import javax.xml.bind.*; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Set; + +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver.ECLIPSELINK_SERIALIZED_TYPES; + +/** + * + * This class is an extension of {@link JSONRootElementProvider} in a sense that + * it replaces the functionality of the + * {@link JSONRootElementProvider#writeTo(Object, MediaType, Charset, Marshaller, OutputStream)} + * with a custom implementation. + * + * The reason why the original + * {@link JSONRootElementProvider#writeTo(Object, MediaType, Charset, Marshaller, OutputStream)} + * is not adequate is that it calls + * {@link JSONJAXBContext#getJSONMarshaller(Marshaller, JAXBContext)} that + * creates a {@link com.sun.jersey.json.impl.BaseJSONMarshaller} in case of the + * marshaller is not an instance of {@link JSONMarshaller}. + * + * Currently there are some POJOs, e.g. {@link SchedulerTypeInfo}, + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo} that + * are serialized with EclipseLink's JAXB implementation, see implementation in + * {@link JAXBContextResolver#createTypesContextMap()}. If a class with + * EclipseLink JAXB is being serialized, the marshaller created by Jersey is not + * an instance of {@link JSONMarshaller} therefore + * {@link com.sun.jersey.api.json.JSONConfiguration#DEFAULT} would be used as a + * JSON config for serialization. + * + * Since the {@link com.sun.jersey.api.json.JSONConfiguration#DEFAULT} is a + * hardcoded JSON config and it turns on root unwrapping, therefore the root tag + * would not have been serialized. + * + * A more simplistic approach would have been to replace + * {@link com.sun.jersey.api.json.JSONConfiguration#DEFAULT} with a more + * feasible configuration (without root unwrapping) but it caused several tests + * to fail since some webservice relied on the default configuration. + * + * Every operation except + * {@link JSONRootElementProviderEclipseLink#writeTo(Object, MediaType, Charset, Marshaller, OutputStream)} + * is delegated to {@link JSONRootElementProvider} methods, respectively. + * Subclassing the delegate is not possible as constructors are package-private. + * + * The only special thing is in + * {@link App#isWriteable(Class, Type, Annotation[], MediaType)} that only + * treats a passed-in type as supported if it is included in the + * {@link App#SUPPORTED_TYPES} set. This way, we can limit the serialization + * representation's scope to such supported classes. Delegate method + * {@link JSONRootElementProviderEclipseLink#readFrom(Class, MediaType, Unmarshaller, InputStream)} + * is treated special since this method is protected on the delagate so the call + * should utilize reflection. + * + */ +public class JSONRootElementProviderEclipseLink + extends AbstractRootElementProvider { + private static final Logger LOG = + LoggerFactory.getLogger(JSONRootElementProviderEclipseLink.class); + + private final JSONRootElementProvider delegate; + + JSONRootElementProviderEclipseLink(Providers ps) throws Exception { + super(ps); + this.delegate = createDelegateInstance(ps); + } + + JSONRootElementProviderEclipseLink(Providers ps, MediaType mt) + throws Exception { + super(ps, mt); + this.delegate = createDelegateInstance(ps, mt); + } + + /** + * Since JSONRootElementProvider constructors are package-private, it cannot + * be instantiated simply, but with reflection. + * + * @param providers + * @return + */ + private static JSONRootElementProvider createDelegateInstance( + Providers providers) throws Exception { + + Constructor constructor = + JSONRootElementProvider.class.getDeclaredConstructor(Providers.class); + constructor.setAccessible(true); + return constructor.newInstance(providers); + } + + /** + * Since JSONRootElementProvider constructors are package-private, it cannot + * be instantiated simply, but with reflection. + * + * @param providers + * @return + */ + private static JSONRootElementProvider createDelegateInstance( + Providers providers, MediaType mediaType) throws Exception { + + Constructor constructor = + JSONRootElementProvider.class.getDeclaredConstructor(Providers.class, + MediaType.class); + constructor.setAccessible(true); + return constructor.newInstance(providers, mediaType); + } + + @Context + @Override + public void setConfiguration(FeaturesAndProperties fp) { + delegate.setConfiguration(fp); + } + + @Override + public boolean isReadable(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return delegate.isReadable(type, genericType, annotations, mediaType); + } + + @Override + public boolean isWriteable(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return delegate.isWriteable(type, genericType, annotations, mediaType); + } + + @Override + protected final Object readFrom(Class type, MediaType mediaType, + Unmarshaller u, InputStream entityStream) throws JAXBException { + try { + return invokeReadFromOnDelegate(type, mediaType, u, entityStream); + } catch (InvocationTargetException e) { + if (e.getCause() instanceof JAXBException) { + throw (JAXBException) e.getCause(); + } else { + throw new RuntimeException(e); + } + } catch (NoSuchMethodException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("JavaReflectionMemberAccess") + private Object invokeReadFromOnDelegate(Class type, + MediaType mediaType, Unmarshaller u, + InputStream entityStream) + throws NoSuchMethodException, InvocationTargetException, + IllegalAccessException { + Method method = JSONRootElementProvider.class.getMethod("readFrom", + Class.class, MediaType.class, Unmarshaller.class, InputStream.class); + method.setAccessible(true); + return method.invoke(delegate, type, mediaType, u, entityStream); + } + + /** + * Concrete implementation of a {@link javax.ws.rs.ext.MessageBodyWriter}. + * This implementation is used for serializing objects that contain resources + * in a map and their representation needs to be flattened. + * See this class as an example: + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfoWithCustomResourceTypes} + */ + @Produces("application/json") + @Consumes("application/json") + public static final class App extends JSONRootElementProviderEclipseLink { + private static final Set SUPPORTED_TYPES = + ImmutableSet.copyOf(ECLIPSELINK_SERIALIZED_TYPES); + + public App(@Context Providers ps) throws Exception { + super(ps, MediaType.APPLICATION_JSON_TYPE); + LOG.debug("Creating {} , supported types: {}", + getClass(), Arrays.toString(SUPPORTED_TYPES.toArray())); + } + + @Override + public boolean isWriteable(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return isClassSupported(type) + && super.isWriteable(type, genericType, annotations, mediaType); + } + + private boolean isClassSupported(Class type) { + return SUPPORTED_TYPES.contains(type); + } + + @Override + protected void writeTo(Object obj, MediaType mediaType, Charset c, + Marshaller defaultMarshaller, OutputStream entityStream) + throws JAXBException { + LOG.debug("Invoked {}.writeTo() with object type {} media type: {}", + getClass(), obj.getClass(), mediaType); + final Marshaller marshaller = + getMarshallerForMediaType(getMediaType(mediaType), defaultMarshaller); + marshaller.marshal(obj, new OutputStreamWriter(entityStream, c)); + } + + private Marshaller getMarshallerForMediaType(MediaType mediaType, + Marshaller defaultMarshaller) { + // fallback to framework-provided marshaller if mediaType is not found in + // map + return MarshallersHolder.getOrDefault(mediaType, defaultMarshaller); + } + + /** + * Gets the MediaType. If the MediaType is an instance of + * QualitySourceMediaType, e.g. application/json;charset=utf-8, it will + * extract the MediaType for application/json. + * + * @param mediaType The default mediaType + * @return + */ + private MediaType getMediaType(final MediaType mediaType) { + if (mediaType instanceof QualitySourceMediaType) { + QualitySourceMediaType qsMediaType = (QualitySourceMediaType) mediaType; + String type = qsMediaType.getType(); + String subtype = qsMediaType.getSubtype(); + return MediaType.valueOf(type + "/" + subtype); + } + return mediaType; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MapAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MapAdapter.java new file mode 100644 index 00000000000..1f4a93ac6d6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MapAdapter.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; +import java.util.*; +import java.util.Map.Entry; +import javax.xml.bind.annotation.*; +import javax.xml.bind.annotation.adapters.XmlAdapter; +import org.eclipse.persistence.oxm.annotations.XmlVariableNode; + +/** + * An adapter class that can map a {@link Map} such that keys become the node + * names. + * See this blogpost for details: + * http://blog.bdoughan.com/2013/06/moxys-xmlvariablenode-using-maps-key-as.html + */ +public class MapAdapter + extends XmlAdapter> { + + /** + * The ValueType of the {@link Map}, meaning that entries will hold the + * {@link AdaptedEntry} objects, these are actually the key/value pairs of the + * map. + */ + public static class AdaptedMap { + + @XmlVariableNode("key") + private List entries = new ArrayList<>(); + + public List getEntries() { + return entries; + } + } + + /** + * Instances of this class represents a key/value pair of a map. + */ + public static class AdaptedEntry { + + @XmlTransient + private String key; + + @XmlValue + private Long value; + + public String getKey() { + return key; + } + + public Long getValue() { + return value; + } + } + + @Override + public AdaptedMap marshal(Map map) { + AdaptedMap adaptedMap = new AdaptedMap(); + + if (map == null) { + return adaptedMap; + } + for (Entry entry : map.entrySet()) { + AdaptedEntry adaptedEntry = new AdaptedEntry(); + adaptedEntry.key = entry.getKey(); + adaptedEntry.value = entry.getValue(); + adaptedMap.entries.add(adaptedEntry); + } + return adaptedMap; + } + + @Override + public Map unmarshal(AdaptedMap adaptedMap) { + List adaptedEntries = adaptedMap.entries; + Map map = new HashMap<>(adaptedEntries.size()); + for (AdaptedEntry adaptedEntry : adaptedEntries) { + map.put(adaptedEntry.key, adaptedEntry.value); + } + return map; + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MarshallersHolder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MarshallersHolder.java new file mode 100644 index 00000000000..70fca68b213 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MarshallersHolder.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.eclipse.persistence.jaxb.MarshallerProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.MediaType; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import java.util.Arrays; +import java.util.Map; + +import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; +import static javax.ws.rs.core.MediaType.APPLICATION_XML_TYPE; + +/** + * This class holds json and xml marshallers for their respective media types. + * Since the default configuration of marshallers for EclipseLink JAXB provider + * was not adequate for every situation, here we have them configured properly. + * The method {@link MarshallersHolder#getOrDefault(MediaType, Marshaller)} + * should only be used for EclipseLink serialization, see the only usage from + * @link {JSONRootElementProviderEclipseLink.App#writeTo(java.lang.Object, javax.ws.rs.core.MediaType, java.nio.charset.Charset, javax.xml.bind.Marshaller, java.io.OutputStream)}. + */ +public class MarshallersHolder { + private static final Logger LOG = LoggerFactory.getLogger(MarshallersHolder.class); + private static final Map MARSHALLERS = + createMarshallers(); + + private static ImmutableMap createMarshallers() { + return ImmutableMap.of(APPLICATION_JSON_TYPE, + createMarshallerForMediaType(APPLICATION_JSON_TYPE), + APPLICATION_XML_TYPE, + createMarshallerForMediaType(APPLICATION_XML_TYPE)); + } + + private static Marshaller createMarshallerForMediaType(MediaType mediaType) { + try { + // use an arbitrary EclipseLink-serialized class to get EclipseLink's + // JAXBContext + JAXBContext jaxbContext = createJaxbContext( + JAXBContextResolver.ECLIPSELINK_SERIALIZED_TYPES.get(0)); + + if (mediaType.equals(APPLICATION_JSON_TYPE)) { + return createJsonMarshaller(jaxbContext); + } else if (mediaType.equals(APPLICATION_XML_TYPE)) { + return createXmlMarshaller(jaxbContext); + } else { + throw new IllegalArgumentException("Unknown mediaType: " + mediaType); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static JAXBContext createJaxbContext(Class clazz) throws Exception { + JAXBContextResolver jaxbContextResolver = new JAXBContextResolver(); + return jaxbContextResolver.getContext(clazz); + } + + public static Marshaller createJsonMarshaller(JAXBContext jaxbContext) + throws JAXBException { + Marshaller marshaller = jaxbContext.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + marshaller.setProperty(MarshallerProperties.MEDIA_TYPE, "application/json"); + marshaller.setProperty(MarshallerProperties.JSON_INCLUDE_ROOT, true); + marshaller.setProperty(MarshallerProperties.JSON_MARSHAL_EMPTY_COLLECTIONS, + false); + + LOG.debug("Created JSON marshaller (EclipseLink) {}", marshaller); + return marshaller; + } + + public static Marshaller createXmlMarshaller(JAXBContext jaxbContext) + throws JAXBException { + Marshaller marshaller = jaxbContext.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + + return marshaller; + } + + public static Marshaller getOrDefault(MediaType mediaType, + Marshaller defaultMarshaller) { + LOG.debug( + "Invoking {}.getOrDefault() with media type: {}, available marshallers: {}", + MarshallersHolder.class, mediaType, + Arrays.toString(MARSHALLERS.entrySet().toArray())); + return MARSHALLERS.getOrDefault(mediaType, defaultMarshaller); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/package-info.java new file mode 100644 index 00000000000..860ec135535 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains classes for EclipseLink JAXB data serialization. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/META-INF/services/javax.ws.rs.ext.MessageBodyWriter b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/META-INF/services/javax.ws.rs.ext.MessageBodyWriter new file mode 100644 index 00000000000..40efb19e312 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/META-INF/services/javax.ws.rs.ext.MessageBodyWriter @@ -0,0 +1 @@ +org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.JSONRootElementProviderEclipseLink$App \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 481645bb494..eb3a03bd0fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -48,6 +48,9 @@ import org.junit.Assert; import org.junit.Test; +/** + * Tests fair scheduler configuration. + */ public class TestFairSchedulerConfiguration { private static final String A_CUSTOM_RESOURCE = "a-custom-resource"; @@ -157,12 +160,12 @@ public void testParseResourceConfigValue() throws Exception { parseResourceConfigValue("10.9% memory, 50.6% cpu"). getResource(clusterResource)); } - + @Test(expected = AllocationConfigurationException.class) public void testNoUnits() throws Exception { parseResourceConfigValue("1024"); } - + @Test(expected = AllocationConfigurationException.class) public void testOnlyMemory() throws Exception { parseResourceConfigValue("1024mb"); @@ -172,7 +175,7 @@ public void testOnlyMemory() throws Exception { public void testOnlyCPU() throws Exception { parseResourceConfigValue("1024vcores"); } - + @Test(expected = AllocationConfigurationException.class) public void testGibberish() throws Exception { parseResourceConfigValue("1o24vc0res"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppInfoVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppInfoVerifications.java new file mode 100644 index 00000000000..3b2c75bbaa4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppInfoVerifications.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; + +import static org.junit.Assert.*; + +/** + * Contains all value verifications that are needed to verify in {@link AppInfo} + * representation objects. + */ +public class AppInfoVerifications { + + /** + * Tests whether {@link AppInfo} representation object contains the required + * values as per defined in the specified app parameter. + * @param app an RMApp instance that contains the required values to test against. + */ + public static void verify(ElementWrapper info, RMApp app) { + WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() + .toString(), info.getString("id")); + WebServicesTestUtils.checkStringMatch("user", app.getUser(), + info.getString("user")); + WebServicesTestUtils.checkStringMatch("name", app.getName(), + info.getString("name")); + WebServicesTestUtils.checkStringMatch("applicationType", + app.getApplicationType(), info.getString("applicationType")); + WebServicesTestUtils.checkStringMatch("queue", app.getQueue(), + info.getString("queue")); + assertEquals("priority doesn't match", 0, (int) info.getInt("priority")); + WebServicesTestUtils.checkStringMatch("state", app.getState().toString(), + info.getString("state")); + WebServicesTestUtils.checkStringMatch("finalStatus", app + .getFinalApplicationStatus().toString(), + info.getString("finalStatus")); + assertEquals("progress doesn't match", 0, info.getFloat("progress"), 0.0); + if ("UNASSIGNED".equals(info.getString("trackingUI"))) { + WebServicesTestUtils.checkStringMatch("trackingUI", "UNASSIGNED", + info.getString("trackingUI")); + } + WebServicesTestUtils.checkStringEqual("diagnostics", + app.getDiagnostics().toString(), info.getString("diagnostics")); + assertEquals("clusterId doesn't match", + ResourceManager.getClusterTimeStamp(), + (long)info.getLong("clusterId")); + assertEquals("startedTime doesn't match", app.getStartTime(), + (long)info.getLong("startedTime")); + assertEquals("finishedTime doesn't match", app.getFinishTime(), + (long)info.getLong("finishedTime")); + assertTrue("elapsed time not greater than 0", + info.getLong("elapsedTime") > 0); + WebServicesTestUtils.checkStringMatch("amHostHttpAddress", app + .getCurrentAppAttempt().getMasterContainer() + .getNodeHttpAddress(), + info.getString("amHostHttpAddress")); + assertTrue("amContainerLogs doesn't match", + info.getString("amContainerLogs").startsWith("http://")); + assertTrue("amContainerLogs doesn't contain user info", + info.getString("amContainerLogs").endsWith("/" + app.getUser())); + assertEquals("allocatedMB doesn't match", 1024, + (int)info.getInt("allocatedMB")); + assertEquals("allocatedVCores doesn't match", 1, + (int)info.getInt("allocatedVCores")); + assertEquals("queueUsagePerc doesn't match", 50.0f, + info.getFloat("queueUsagePercentage"), 0.01f); + assertEquals("clusterUsagePerc doesn't match", 50.0f, + info.getFloat("clusterUsagePercentage"), 0.01f); + assertEquals("numContainers doesn't match", 1, (int) + info.getInt("runningContainers")); + assertNotNull("preemptedResourceSecondsMap should not be null", + info.getChild("preemptedResourceSecondsMap")); + assertEquals("preemptedResourceMB doesn't match", app + .getRMAppMetrics().getResourcePreempted().getMemorySize(), + (int)info.getInt("preemptedResourceMB")); + assertEquals("preemptedResourceVCores doesn't match", app + .getRMAppMetrics().getResourcePreempted().getVirtualCores(), + (int) info.getInt("preemptedResourceVCores")); + assertEquals("numNonAMContainerPreempted doesn't match", app + .getRMAppMetrics().getNumNonAMContainersPreempted(), + (int)info.getInt("numNonAMContainerPreempted")); + assertEquals("numAMContainerPreempted doesn't match", app + .getRMAppMetrics().getNumAMContainersPreempted(), + (int) info.getInt("numAMContainerPreempted")); + assertEquals("Log aggregation Status doesn't match", app + .getLogAggregationStatusForAppReport().toString(), + info.getString("logAggregationStatus")); + assertEquals("unmanagedApplication doesn't match", app + .getApplicationSubmissionContext().getUnmanagedAM(), + info.getBoolean("unmanagedApplication")); + assertEquals("unmanagedApplication doesn't match", + app.getApplicationSubmissionContext().getNodeLabelExpression(), + info.getStringSafely("appNodeLabelExpression")); + assertEquals("unmanagedApplication doesn't match", + app.getAMResourceRequests().get(0).getNodeLabelExpression(), + info.getStringSafely("amNodeLabelExpression")); + assertEquals("amRPCAddress", + AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()), + info.getStringSafely("amRPCAddress")); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ResourceRequestsVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ResourceRequestsVerifications.java new file mode 100644 index 00000000000..d8fa89b7ce2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ResourceRequestsVerifications.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import com.google.common.collect.Lists; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Performs value verifications on + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo} + * objects agains the values of {@link ResourceRequest}. With the help of the + * {@link Builder}, users can also make verifications of the custom resource + * types and its values. + */ +public class ResourceRequestsVerifications { + private final ResourceRequest resourceRequest; + private final ElementWrapper requestInfo; + private final Map customResourceTypes; + private final List expectedCustomResourceTypes; + + ResourceRequestsVerifications(Builder builder) { + this.resourceRequest = builder.resourceRequest; + this.requestInfo = builder.requestInfo; + this.customResourceTypes = builder.customResourceTypes; + this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes; + } + + public static void verify(ElementWrapper requestInfo, ResourceRequest rr) { + createDefaultBuilder(requestInfo, rr) + .build() + .verify(); + } + + public static void verifyWithCustomResourceTypes(ElementWrapper requestInfo, + ResourceRequest resourceRequest, List expectedResourceTypes) { + + createDefaultBuilder(requestInfo, resourceRequest) + .withExpectedCustomResourceTypes(expectedResourceTypes) + .withCustomResourceTypes(extractActualCustomResourceTypes(requestInfo, + expectedResourceTypes)) + .build() + .verify(); + } + + private static Builder createDefaultBuilder(ElementWrapper requestInfo, + ResourceRequest resourceRequest) { + return new ResourceRequestsVerifications.Builder() + .withRequest(resourceRequest) + .withRequestInfo(requestInfo); + } + + private static Map extractActualCustomResourceTypes( + ElementWrapper requestInfo, List expectedResourceTypes) { + ElementWrapper capability = requestInfo.getChild("capability"); + + return expectedResourceTypes.stream() + .collect(HashMap::new, + (map, value) -> map.put(value, + extractCustomResorceTypeValue(capability, value)), + HashMap::putAll); + } + + private static Long extractCustomResorceTypeValue(ElementWrapper capability, + String resourceType) { + if (capability.hasChild(resourceType)) { + return capability.getLong(resourceType); + } + return null; + } + + private void verify() { + assertEquals("nodeLabelExpression doesn't match", + resourceRequest.getNodeLabelExpression(), + requestInfo.getString("nodeLabelExpression")); + assertEquals("numContainers doesn't match", + resourceRequest.getNumContainers(), + (int)requestInfo.getInt("numContainers")); + assertEquals("relaxLocality doesn't match", + resourceRequest.getRelaxLocality(), + requestInfo.getBoolean("relaxLocality")); + assertEquals("priority does not match", + resourceRequest.getPriority().getPriority(), + (int)requestInfo.getInt("priority")); + assertEquals("resourceName does not match", + resourceRequest.getResourceName(), + requestInfo.getString("resourceName")); + assertEquals("memory does not match", + resourceRequest.getCapability().getMemorySize(), + (long)requestInfo.getChild("capability").getLong("memory")); + assertEquals("vCores does not match", + resourceRequest.getCapability().getVirtualCores(), + (long)requestInfo.getChild("capability").getLong("vCores")); + + for (String expectedCustomResourceType : expectedCustomResourceTypes) { + assertTrue( + "Custom resource type " + expectedCustomResourceType + + " cannot be found!", + customResourceTypes.containsKey(expectedCustomResourceType)); + + Long resourceValue = customResourceTypes.get(expectedCustomResourceType); + assertNotNull("Resource value should not be null!", resourceValue); + } + + ElementWrapper executionTypeRequest = + requestInfo.getChild("executionTypeRequest"); + assertEquals("executionType does not match", + resourceRequest.getExecutionTypeRequest().getExecutionType().name(), + executionTypeRequest.getString("executionType")); + assertEquals("enforceExecutionType does not match", + resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(), + executionTypeRequest.getBoolean("enforceExecutionType")); + } + + /** + * Builder class for {@link ResourceRequestsVerifications}. + */ + public static final class Builder { + private List expectedCustomResourceTypes = Lists.newArrayList(); + private Map customResourceTypes; + private ResourceRequest resourceRequest; + private ElementWrapper requestInfo; + + Builder() { + } + + public static Builder create() { + return new Builder(); + } + + public Builder withExpectedCustomResourceTypes( + List expectedCustomResourceTypes) { + this.expectedCustomResourceTypes = expectedCustomResourceTypes; + return this; + } + + public Builder withCustomResourceTypes( + Map customResourceTypes) { + this.customResourceTypes = customResourceTypes; + return this; + } + + public Builder withRequest(ResourceRequest resourceRequest) { + this.resourceRequest = resourceRequest; + return this; + } + + public Builder withRequestInfo(ElementWrapper requestInfo) { + this.requestInfo = requestInfo; + return this; + } + + public ResourceRequestsVerifications build() { + return new ResourceRequestsVerifications(this); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestMarshalAppInfoWithEclipseLinkJaxbProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestMarshalAppInfoWithEclipseLinkJaxbProvider.java new file mode 100644 index 00000000000..878bfd62322 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestMarshalAppInfoWithEclipseLinkJaxbProvider.java @@ -0,0 +1,310 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; +import org.apache.hadoop.http.JettyUtils; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.*; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ErrorCollector; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import javax.ws.rs.core.MediaType; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.function.Consumer; + +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver.JSON_CONFIG_DEFAULT; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.MarshallersHolder.createJsonMarshaller; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.MarshallersHolder.createXmlMarshaller; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A parameterized test that tests whether collections marshalling works with + * the EclipseLink JAXB provider. Since this class is Parameterized, the second + * run provided by the method that defined the parameters ( + * {@link TestMarshalAppInfoWithEclipseLinkJaxbProvider#marshallers()}) do a + * regression-like test, ensuring that collection marshalling still works fine + * with the normal JAXB context and its marshallers (part of JDK) as well. + */ +@RunWith(Parameterized.class) +public class TestMarshalAppInfoWithEclipseLinkJaxbProvider { + + @Rule + public ErrorCollector collector = new ErrorCollector(); + + @Parameters + public static Collection marshallers() throws Exception { + JAXBContext eclipseLinkJaxbContext = createEclipseLinkJaxbContext(); + Marshaller xmlMarshaller = createXmlMarshaller(eclipseLinkJaxbContext); + Marshaller jsonMarshaller = createJsonMarshaller(eclipseLinkJaxbContext); + + JAXBContext normalJaxbContext = createNormalJaxbContext(); + Marshaller normalXmlMarshaller = + createNormalXmlMarshaller(normalJaxbContext); + Marshaller normalJsonMarshaller = + createNormalJsonMarshaller(normalJaxbContext); + + return Arrays.asList(new Object[][] { { xmlMarshaller, jsonMarshaller }, + { normalXmlMarshaller, normalJsonMarshaller } }); + } + + private Marshaller xmlMarshaller; + private Object jsonMarshaller; + + public TestMarshalAppInfoWithEclipseLinkJaxbProvider(Marshaller xmlMarshaller, + Object jsonMarshaller) { + this.xmlMarshaller = xmlMarshaller; + this.jsonMarshaller = jsonMarshaller; + } + + private static JAXBContext createEclipseLinkJaxbContext() throws Exception { + JAXBContextResolver jaxbContextResolver = new JAXBContextResolver(); + return jaxbContextResolver.getContext(AppInfo.class); + } + + private static JAXBContext createNormalJaxbContext() throws JAXBException { + return new JSONJAXBContext(JSON_CONFIG_DEFAULT, AppInfo.class); + } + + private static Marshaller createNormalXmlMarshaller(JAXBContext jaxbContext) + throws JAXBException { + return jaxbContext.createMarshaller(); + } + + private static Marshaller createNormalJsonMarshaller( + JAXBContext jaxbContext) throws JAXBException { + return jaxbContext.createMarshaller(); + } + + private AppInfo createAppInfoWithEmptyCollections() { + AppInfo appInfo = new AppInfo(); + appInfo.setPreemptedResourceSecondsMap(Maps.newHashMap()); + appInfo.setResourceSecondsMap(Maps.newHashMap()); + appInfo.setResourceRequests(Lists.newArrayList()); + return appInfo; + } + + private AppInfo createAppInfoWithNotEmptyCollections() { + AppInfo appInfo = new AppInfo(); + appInfo.setPreemptedResourceSecondsMap(ImmutableMap.of("bla", 2L)); + appInfo.setResourceSecondsMap(ImmutableMap.of("bla", 12L)); + appInfo.setResourceRequests(ImmutableList.of(new ResourceRequestInfo())); + return appInfo; + } + + private void assertHasChild(ElementWrapper appInfoWrapper, String child) { + String message = + "AppInfo " + appInfoWrapper + " does not have child: " + child; + collector.checkThat(message, appInfoWrapper.hasChild(child), equalTo(true)); + } + + private void assertDoesNotHaveChild(ElementWrapper appInfoWrapper, + String child) { + String message = + "AppInfo " + appInfoWrapper + " should not have child: " + child; + collector.checkThat(message, appInfoWrapper.hasChild(child), + equalTo(false)); + } + + private void assertChildHasValues(ElementWrapper appInfoWrapper, String child, + Map values) { + collector.checkThat( + "AppInfo " + appInfoWrapper + " does not have child: " + child, + appInfoWrapper.hasChild(child), equalTo(true)); + + ElementWrapper mapKeys = appInfoWrapper.getChild(child); + collector + .checkThat( + String.format("app.%s: %s does not have expected size: %d", child, + mapKeys, values.size()), + mapKeys.length(), equalTo(values.size())); + + for (Map.Entry expectedEntry : values.entrySet()) { + String nameOfMapKey = expectedEntry.getKey(); + Long expectedMapValue = expectedEntry.getValue(); + collector.checkThat(mapKeys.getLong(nameOfMapKey), + equalTo(expectedMapValue)); + } + + } + + private WebResource createMockResource() { + WebResource mockResource = mock(WebResource.class); + when(mockResource.toString()).thenReturn("dummyPath"); + return mockResource; + } + + private BufferedClientResponse createMockResponse(StringWriter sw, + MediaType mediaType) throws JSONException { + BufferedClientResponse mockResponse = mock(BufferedClientResponse.class); + + MediaType mockMediaType = mock(MediaType.class); + when(mockMediaType.toString()) + .thenReturn(mediaType + "; " + JettyUtils.UTF_8); + + when(mockResponse.getType()).thenReturn(mockMediaType); + when(mockResponse.getEntity(eq(String.class))).thenReturn(sw.toString()); + + if (mediaType.equals(MediaType.APPLICATION_JSON_TYPE)) { + JSONObject jsonObject = new JSONObject(sw.toString()); + when(mockResponse.getEntity(eq(JSONObject.class))).thenReturn(jsonObject); + } + + return mockResponse; + } + + private Consumer emptyCollectionsVerifier() { + return responseAdapter -> { + ElementWrapper appInfoWrapper = responseAdapter.getElement("app"); + + assertHasChild(appInfoWrapper, "preemptedResourceSecondsMap"); + assertHasChild(appInfoWrapper, "resourceSecondsMap"); + assertDoesNotHaveChild(appInfoWrapper, "resourceRequests"); + }; + } + + private Consumer notEmptyCollectionsVerifier() { + return responseAdapter -> { + ElementWrapper appInfoWrapper = responseAdapter.getElement("app"); + + assertChildHasValues(appInfoWrapper, "preemptedResourceSecondsMap", + ImmutableMap.of("bla", 2L)); + assertChildHasValues(appInfoWrapper, "resourceSecondsMap", + ImmutableMap.of("bla", 12L)); + assertHasChild(appInfoWrapper, "resourceRequests"); + }; + } + + private void marshalJson(AppInfo appInfo, StringWriter sw) + throws JAXBException { + if (jsonMarshaller instanceof JSONMarshaller) { + ((JSONMarshaller) jsonMarshaller).marshallToJSON(appInfo, sw); + } else if (jsonMarshaller instanceof Marshaller) { + ((Marshaller) jsonMarshaller).marshal(appInfo, sw); + } else { + throw new IllegalStateException( + "Wrong type of JSON marshaller: " + jsonMarshaller.getClass()); + } + } + + @Test + public void testMarshalEmptyCollectionsXmlEclipseLinkJaxbContext() + throws Exception { + AppInfo appInfo = createAppInfoWithEmptyCollections(); + StringWriter sw = new StringWriter(); + xmlMarshaller.marshal(appInfo, sw); + + WebResource mockResource = createMockResource(); + BufferedClientResponse mockResponse = + createMockResponse(sw, MediaType.APPLICATION_XML_TYPE); + + XmlCustomResourceTypeTestCase testCase = + new XmlCustomResourceTypeTestCase(mockResource, mockResponse); + testCase.verify(emptyCollectionsVerifier()); + } + + /** + * Note: Does not make sense to check normal jersey Marshaller with not empty + * collections on {@link AppInfo} since + * @XmlJavaTypeAdapter and @XmlPath are altering the marshalling behavior that + * the Jersey Marshaller could not handle. + * @throws Exception + */ + @Test + public void testMarshalNotEmptyCollectionsXml() throws Exception { + JAXBContext eclipseLinkJaxbContext = createEclipseLinkJaxbContext(); + Marshaller xmlMarshaller = createXmlMarshaller(eclipseLinkJaxbContext); + + AppInfo appInfo = createAppInfoWithNotEmptyCollections(); + StringWriter sw = new StringWriter(); + xmlMarshaller.marshal(appInfo, sw); + + WebResource mockResource = createMockResource(); + BufferedClientResponse mockResponse = + createMockResponse(sw, MediaType.APPLICATION_XML_TYPE); + + XmlCustomResourceTypeTestCase testCase = + new XmlCustomResourceTypeTestCase(mockResource, mockResponse); + testCase.verify(notEmptyCollectionsVerifier()); + } + + + @Test + public void testMarshalEmptyCollectionsJson() throws Exception { + AppInfo appInfo = createAppInfoWithEmptyCollections(); + StringWriter sw = new StringWriter(); + marshalJson(appInfo, sw); + + WebResource mockResource = createMockResource(); + BufferedClientResponse mockResponse = + createMockResponse(sw, MediaType.APPLICATION_JSON_TYPE); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(mockResource, mockResponse); + testCase.verify(emptyCollectionsVerifier()); + } + + /** + * Note: does not make sense to check normal jersey Marshaller with not empty + * collections on {@link AppInfo} since + * @XmlJavaTypeAdapter and @XmlPath are altering the marshalling behavior that + * the Jersey Marshaller could not handle. + * @throws Exception + */ + @Test + public void testMarshalNotEmptyCollectionsJson() throws Exception { + JAXBContext eclipseLinkJaxbContext = createEclipseLinkJaxbContext(); + Marshaller jsonMarshaller = createJsonMarshaller(eclipseLinkJaxbContext); + + AppInfo appInfo = createAppInfoWithNotEmptyCollections(); + StringWriter sw = new StringWriter(); + jsonMarshaller.marshal(appInfo, sw); + + WebResource mockResource = createMockResource(); + BufferedClientResponse mockResponse = + createMockResponse(sw, MediaType.APPLICATION_JSON_TYPE); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(mockResource, mockResponse); + testCase.verify(notEmptyCollectionsVerifier()); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index f93a3fc3540..5fe76d58224 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -50,11 +50,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; -import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.*; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -75,6 +71,8 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -90,6 +88,8 @@ import com.sun.jersey.test.framework.WebAppDescriptor; public class TestRMWebServices extends JerseyTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(TestRMWebServices.class); private static MockRM rm; @@ -466,19 +466,19 @@ public void verifyClusterMetrics(int submittedApps, int completedApps, QueueMetrics metrics = rs.getRootQueueMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); - long totalMBExpect = + long totalMBExpect = metrics.getAvailableMB() + metrics.getAllocatedMB(); - long totalVirtualCoresExpect = + long totalVirtualCoresExpect = metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores(); - assertEquals("appsSubmitted doesn't match", + assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), submittedApps); - assertEquals("appsCompleted doesn't match", + assertEquals("appsCompleted doesn't match", metrics.getAppsCompleted(), completedApps); assertEquals("reservedMB doesn't match", metrics.getReservedMB(), reservedMB); - assertEquals("availableMB doesn't match", + assertEquals("availableMB doesn't match", metrics.getAvailableMB(), availableMB); - assertEquals("allocatedMB doesn't match", + assertEquals("allocatedMB doesn't match", metrics.getAllocatedMB(), allocMB); assertEquals("reservedVirtualCores doesn't match", metrics.getReservedVirtualCores(), reservedVirtualCores); @@ -591,11 +591,13 @@ public void verifySchedulerFifoXML(String xml) throws JSONException, public void verifyClusterSchedulerFifo(JSONObject json) throws JSONException, Exception { - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("incorrect number of elements in: " + json, 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); - assertEquals("incorrect number of elements", 1, info.length()); + assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 11, info.length()); + + LOG.debug("schedulerInfo: {}", info); + assertEquals("incorrect number of elements in: " + info, 11, info.length()); verifyClusterSchedulerFifoGeneric(info.getString("type"), info.getString("qstate"), (float) info.getDouble("capacity"), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index f0704ac99e6..b2128c2ac35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -52,6 +52,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp + .representationhelper.*; +import org.apache.hadoop.yarn.server.resourcemanager.webapp + .representationhelper.json.JsonObjectWrapper; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -79,7 +83,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { private static MockRM rm; - + private static final int CONTAINER_MB = 1024; private static class WebServletModule extends ServletModule { @@ -157,27 +161,27 @@ public void testAppsXML() throws JSONException, Exception { RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").accept(MediaType.APPLICATION_XML) + WebResource path = r.path("ws").path("v1").path("cluster") + .path("apps"); + ClientResponse response = path.accept(MediaType.APPLICATION_XML) .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(xml)); - Document dom = db.parse(is); - NodeList nodesApps = dom.getElementsByTagName("apps"); - assertEquals("incorrect number of elements", 1, nodesApps.getLength()); - NodeList nodes = dom.getElementsByTagName("app"); - assertEquals("incorrect number of elements", 1, nodes.getLength()); - verifyAppsXML(nodes, app1, false); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + ArrayWrapper arrayWrapper = responseAdapter.getArray("apps[]"); + assertEquals("incorrect number of elements", 1, arrayWrapper.length()); + + ArrayWrapper appArray = responseAdapter.getArray("apps.app[]"); + assertEquals("incorrect number of elements", 1, appArray.length()); + verifyAppsXML(appArray, app1, false); + }); + rm.stop(); } @Test - public void testRunningApp() throws JSONException, Exception { + public void testRunningApp() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); @@ -186,22 +190,23 @@ public void testRunningApp() throws JSONException, Exception { amNodeManager.nodeHeartbeat(true); WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").accept(MediaType.APPLICATION_XML) + WebResource path = r.path("ws").path("v1").path("cluster") + .path("apps"); + ClientResponse response = path.accept(MediaType.APPLICATION_XML) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); - String xml = response.getEntity(String.class); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(xml)); - Document dom = db.parse(is); - NodeList nodesApps = dom.getElementsByTagName("apps"); - assertEquals("incorrect number of elements", 1, nodesApps.getLength()); - NodeList nodes = dom.getElementsByTagName("app"); - assertEquals("incorrect number of elements", 1, nodes.getLength()); - verifyAppsXML(nodes, app1, true); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + ElementWrapper appsWrapper = responseAdapter.getElement("apps"); + assertEquals("incorrect number of elements", 1, appsWrapper.length()); + + ArrayWrapper appArray = appsWrapper.getChildArray("app"); + assertEquals("incorrect number of elements", 1, appArray.length()); + verifyAppsXML(appArray, app1, false); + }); testAppsHelper("apps/", app1, MediaType.APPLICATION_JSON, true); rm.stop(); @@ -254,8 +259,8 @@ public void testAppsHelper(String path, RMApp app, String media, assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app, hasResourceReq); - + verifyAppInfoJson( + new JsonObjectWrapper(array.getJSONObject(0)), app, hasResourceReq); } @Test @@ -278,7 +283,8 @@ public void testAppsQueryState() throws JSONException, Exception { assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app1, false); + verifyAppInfoJson( + new JsonObjectWrapper(array.getJSONObject(0)), app1, false); rm.stop(); } @@ -324,7 +330,7 @@ public void testAppsQueryStates() throws JSONException, Exception { assertEquals("incorrect number of elements", 1, apps.length()); array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 2, array.length()); - assertTrue("both app states of ACCEPTED and KILLED are not present", + assertTrue("both app states of ACCEPTED and KILLED are not present", (array.getJSONObject(0).getString("state").equals("ACCEPTED") && array.getJSONObject(1).getString("state").equals("KILLED")) || (array.getJSONObject(0).getString("state").equals("KILLED") && @@ -375,12 +381,12 @@ public void testAppsQueryStatesComma() throws JSONException, Exception { assertEquals("incorrect number of elements", 1, apps.length()); array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 2, array.length()); - assertTrue("both app states of ACCEPTED and KILLED are not present", + assertTrue("both app states of ACCEPTED and KILLED are not present", (array.getJSONObject(0).getString("state").equals("ACCEPTED") && array.getJSONObject(1).getString("state").equals("KILLED")) || (array.getJSONObject(0).getString("state").equals("KILLED") && array.getJSONObject(1).getString("state").equals("ACCEPTED"))); - + rm.stop(); } @@ -511,7 +517,8 @@ public void testAppsQueryFinalStatus() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString()) + .path("apps").queryParam("finalStatus", + FinalApplicationStatus.UNDEFINED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); @@ -522,7 +529,8 @@ public void testAppsQueryFinalStatus() throws JSONException, Exception { assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app1, false); + verifyAppInfoJson( + new JsonObjectWrapper(array.getJSONObject(0)), app1, false); rm.stop(); } @@ -1505,7 +1513,8 @@ public void testSingleAppsHelper(String path, RMApp app, String media) JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - verifyAppInfo(json.getJSONObject("app"), app, false); + verifyAppInfoJson( + new JsonObjectWrapper(json.getJSONObject("app")), app, false); } @Test @@ -1514,270 +1523,83 @@ public void testSingleAppsXML() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").path(app1.getApplicationId().toString()) - .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); + WebResource path = r.path("ws").path("v1").path("cluster") + .path("apps").path(app1.getApplicationId().toString()); + ClientResponse response = path.accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + ArrayWrapper arrayWrapper = responseAdapter.getArray("app[]"); + assertEquals("incorrect number of elements", 1, arrayWrapper.length()); + verifyAppsXML(arrayWrapper, app1, false); + }); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(xml)); - Document dom = db.parse(is); - NodeList nodes = dom.getElementsByTagName("app"); - assertEquals("incorrect number of elements", 1, nodes.getLength()); - verifyAppsXML(nodes, app1, false); rm.stop(); } - public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) - throws JSONException, Exception { - - for (int i = 0; i < nodes.getLength(); i++) { - Element element = (Element) nodes.item(i); - - verifyAppInfoGeneric(app, - WebServicesTestUtils.getXmlString(element, "id"), - WebServicesTestUtils.getXmlString(element, "user"), - WebServicesTestUtils.getXmlString(element, "name"), - WebServicesTestUtils.getXmlString(element, "applicationType"), - WebServicesTestUtils.getXmlString(element, "queue"), - WebServicesTestUtils.getXmlInt(element, "priority"), - WebServicesTestUtils.getXmlString(element, "state"), - WebServicesTestUtils.getXmlString(element, "finalStatus"), - WebServicesTestUtils.getXmlFloat(element, "progress"), - WebServicesTestUtils.getXmlString(element, "trackingUI"), - WebServicesTestUtils.getXmlString(element, "diagnostics"), - WebServicesTestUtils.getXmlLong(element, "clusterId"), - WebServicesTestUtils.getXmlLong(element, "startedTime"), - WebServicesTestUtils.getXmlLong(element, "finishedTime"), - WebServicesTestUtils.getXmlLong(element, "elapsedTime"), - WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"), - WebServicesTestUtils.getXmlString(element, "amContainerLogs"), - WebServicesTestUtils.getXmlInt(element, "allocatedMB"), - WebServicesTestUtils.getXmlInt(element, "allocatedVCores"), - WebServicesTestUtils.getXmlInt(element, "runningContainers"), - WebServicesTestUtils.getXmlFloat(element, "queueUsagePercentage"), - WebServicesTestUtils.getXmlFloat(element, "clusterUsagePercentage"), - WebServicesTestUtils.getXmlInt(element, "preemptedResourceMB"), - WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"), - WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"), - WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"), - WebServicesTestUtils.getXmlString(element, "logAggregationStatus"), - WebServicesTestUtils.getXmlBoolean(element, "unmanagedApplication"), - WebServicesTestUtils.getXmlString(element, "appNodeLabelExpression"), - WebServicesTestUtils.getXmlString(element, "amNodeLabelExpression"), - WebServicesTestUtils.getXmlString(element, "amRPCAddress")); + private void verifyAppsXML(ArrayWrapper array, RMApp app, boolean + hasResourceReq) { + for (int i = 0; i < array.length(); i++) { + ElementWrapper element = array.getObjectAtIndex(i); + AppInfoVerifications.verify(element, app); if (hasResourceReq) { - assertEquals(element.getElementsByTagName("resourceRequests").getLength(), - 1); - Element resourceRequests = - (Element) element.getElementsByTagName("resourceRequests").item(0); - Element capability = - (Element) resourceRequests.getElementsByTagName("capability").item(0); + ArrayWrapper resourceRequests = element + .getChildArray("resourceRequests"); + assertEquals(resourceRequests.length(), 1); + ElementWrapper resourceRequest = resourceRequests.getObjectAtIndex(0); ResourceRequest rr = ((AbstractYarnScheduler)rm.getRMContext().getScheduler()) .getApplicationAttempt( app.getCurrentAppAttempt().getAppAttemptId()) .getAppSchedulingInfo().getAllResourceRequests().get(0); - verifyResourceRequestsGeneric(rr, - WebServicesTestUtils.getXmlString(resourceRequests, - "nodeLabelExpression"), - WebServicesTestUtils.getXmlInt(resourceRequests, "numContainers"), - WebServicesTestUtils.getXmlBoolean(resourceRequests, "relaxLocality"), - WebServicesTestUtils.getXmlInt(resourceRequests, "priority"), - WebServicesTestUtils.getXmlString(resourceRequests, "resourceName"), - WebServicesTestUtils.getXmlLong(capability, "memory"), - WebServicesTestUtils.getXmlLong(capability, "vCores"), - WebServicesTestUtils.getXmlString(resourceRequests, "executionType"), - WebServicesTestUtils.getXmlBoolean(resourceRequests, - "enforceExecutionType")); + ResourceRequestsVerifications.verify(resourceRequest, rr); } } } - public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) - throws JSONException, Exception { + private void verifyAppInfoJson(ElementWrapper info, RMApp app, boolean + hasResourceReqs) { + assertEquals("incorrect number of elements", + getExpectedNumberOfElements(app, hasResourceReqs), info.length()); + AppInfoVerifications.verify(info, app); + if (hasResourceReqs) { + ArrayWrapper resourceRequests = info.getChildArray("resourceRequests"); + ElementWrapper requestInfo = resourceRequests.getObjectAtIndex(0); + ResourceRequest resourceRequest = + ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) + .getApplicationAttempt( + app.getCurrentAppAttempt().getAppAttemptId()) + .getAppSchedulingInfo().getAllResourceRequests().get(0); + + ResourceRequestsVerifications.verify(requestInfo, resourceRequest); + } + } + + private int getExpectedNumberOfElements(RMApp app, boolean hasResourceReqs) { int expectedNumberOfElements = 38 + (hasResourceReqs ? 2 : 0); - String appNodeLabelExpression = null; - String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() .getNodeLabelExpression() != null) { expectedNumberOfElements++; - appNodeLabelExpression = info.getString("appNodeLabelExpression"); } if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { expectedNumberOfElements++; - amNodeLabelExpression = info.getString("amNodeLabelExpression"); } - String amRPCAddress = null; + if (AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) { expectedNumberOfElements++; - amRPCAddress = info.getString("amRPCAddress"); } - assertEquals("incorrect number of elements", expectedNumberOfElements, - info.length()); - verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), - info.getString("name"), info.getString("applicationType"), - info.getString("queue"), info.getInt("priority"), - info.getString("state"), info.getString("finalStatus"), - (float) info.getDouble("progress"), info.getString("trackingUI"), - info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), - info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), - info.getString("amContainerLogs"), info.getInt("allocatedMB"), - info.getInt("allocatedVCores"), info.getInt("runningContainers"), - (float) info.getDouble("queueUsagePercentage"), - (float) info.getDouble("clusterUsagePercentage"), - info.getInt("preemptedResourceMB"), - info.getInt("preemptedResourceVCores"), - info.getInt("numNonAMContainerPreempted"), - info.getInt("numAMContainerPreempted"), - info.getString("logAggregationStatus"), - info.getBoolean("unmanagedApplication"), - appNodeLabelExpression, - amNodeLabelExpression, - amRPCAddress); - - if (hasResourceReqs) { - verifyResourceRequests(info.getJSONArray("resourceRequests"), app); - } - } - - public void verifyAppInfoGeneric(RMApp app, String id, String user, - String name, String applicationType, String queue, int prioirty, - String state, String finalStatus, float progress, String trackingUI, - String diagnostics, long clusterId, long startedTime, long finishedTime, - long elapsedTime, String amHostHttpAddress, String amContainerLogs, - int allocatedMB, int allocatedVCores, int numContainers, - float queueUsagePerc, float clusterUsagePerc, - int preemptedResourceMB, int preemptedResourceVCores, - int numNonAMContainerPreempted, int numAMContainerPreempted, - String logAggregationStatus, boolean unmanagedApplication, - String appNodeLabelExpression, String amNodeLabelExpression, - String amRPCAddress) throws JSONException, Exception { - - WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() - .toString(), id); - WebServicesTestUtils.checkStringMatch("user", app.getUser(), user); - WebServicesTestUtils.checkStringMatch("name", app.getName(), name); - WebServicesTestUtils.checkStringMatch("applicationType", - app.getApplicationType(), applicationType); - WebServicesTestUtils.checkStringMatch("queue", app.getQueue(), queue); - assertEquals("priority doesn't match", 0, prioirty); - WebServicesTestUtils.checkStringMatch("state", app.getState().toString(), - state); - WebServicesTestUtils.checkStringMatch("finalStatus", app - .getFinalApplicationStatus().toString(), finalStatus); - assertEquals("progress doesn't match", 0, progress, 0.0); - if ("UNASSIGNED".equals(trackingUI)) { - WebServicesTestUtils.checkStringMatch("trackingUI", "UNASSIGNED", - trackingUI); - } - WebServicesTestUtils.checkStringEqual("diagnostics", - app.getDiagnostics().toString(), diagnostics); - assertEquals("clusterId doesn't match", - ResourceManager.getClusterTimeStamp(), clusterId); - assertEquals("startedTime doesn't match", app.getStartTime(), startedTime); - assertEquals("finishedTime doesn't match", app.getFinishTime(), - finishedTime); - assertTrue("elapsed time not greater than 0", elapsedTime > 0); - WebServicesTestUtils.checkStringMatch("amHostHttpAddress", app - .getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(), - amHostHttpAddress); - assertTrue("amContainerLogs doesn't match", - amContainerLogs.startsWith("http://")); - assertTrue("amContainerLogs doesn't contain user info", - amContainerLogs.endsWith("/" + app.getUser())); - assertEquals("allocatedMB doesn't match", 1024, allocatedMB); - assertEquals("allocatedVCores doesn't match", 1, allocatedVCores); - assertEquals("queueUsagePerc doesn't match", 50.0f, queueUsagePerc, 0.01f); - assertEquals("clusterUsagePerc doesn't match", 50.0f, clusterUsagePerc, 0.01f); - assertEquals("numContainers doesn't match", 1, numContainers); - assertEquals("preemptedResourceMB doesn't match", app - .getRMAppMetrics().getResourcePreempted().getMemorySize(), - preemptedResourceMB); - assertEquals("preemptedResourceVCores doesn't match", app - .getRMAppMetrics().getResourcePreempted().getVirtualCores(), - preemptedResourceVCores); - assertEquals("numNonAMContainerPreempted doesn't match", app - .getRMAppMetrics().getNumNonAMContainersPreempted(), - numNonAMContainerPreempted); - assertEquals("numAMContainerPreempted doesn't match", app - .getRMAppMetrics().getNumAMContainersPreempted(), - numAMContainerPreempted); - assertEquals("Log aggregation Status doesn't match", app - .getLogAggregationStatusForAppReport().toString(), - logAggregationStatus); - assertEquals("unmanagedApplication doesn't match", app - .getApplicationSubmissionContext().getUnmanagedAM(), - unmanagedApplication); - assertEquals("unmanagedApplication doesn't match", - app.getApplicationSubmissionContext().getNodeLabelExpression(), - appNodeLabelExpression); - assertEquals("unmanagedApplication doesn't match", - app.getAMResourceRequests().get(0).getNodeLabelExpression(), - amNodeLabelExpression); - assertEquals("amRPCAddress", - AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()), - amRPCAddress); - } - - public void verifyResourceRequests(JSONArray resourceRequest, RMApp app) - throws JSONException { - JSONObject requestInfo = resourceRequest.getJSONObject(0); - ResourceRequest rr = - ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) - .getApplicationAttempt( - app.getCurrentAppAttempt().getAppAttemptId()) - .getAppSchedulingInfo().getAllResourceRequests().get(0); - verifyResourceRequestsGeneric(rr, - requestInfo.getString("nodeLabelExpression"), - requestInfo.getInt("numContainers"), - requestInfo.getBoolean("relaxLocality"), requestInfo.getInt("priority"), - requestInfo.getString("resourceName"), - requestInfo.getJSONObject("capability").getLong("memory"), - requestInfo.getJSONObject("capability").getLong("vCores"), - requestInfo.getJSONObject("executionTypeRequest") - .getString("executionType"), - requestInfo.getJSONObject("executionTypeRequest") - .getBoolean("enforceExecutionType")); - } - - public void verifyResourceRequestsGeneric(ResourceRequest request, - String nodeLabelExpression, int numContainers, boolean relaxLocality, - int priority, String resourceName, long memory, long vCores, - String executionType, boolean enforceExecutionType) { - assertEquals("nodeLabelExpression doesn't match", - request.getNodeLabelExpression(), nodeLabelExpression); - assertEquals("numContainers doesn't match", request.getNumContainers(), - numContainers); - assertEquals("relaxLocality doesn't match", request.getRelaxLocality(), - relaxLocality); - assertEquals("priority does not match", request.getPriority().getPriority(), - priority); - assertEquals("resourceName does not match", request.getResourceName(), - resourceName); - assertEquals("memory does not match", - request.getCapability().getMemorySize(), memory); - assertEquals("vCores does not match", - request.getCapability().getVirtualCores(), vCores); - assertEquals("executionType does not match", - request.getExecutionTypeRequest().getExecutionType().name(), - executionType); - assertEquals("enforceExecutionType does not match", - request.getExecutionTypeRequest().getEnforceExecutionType(), - enforceExecutionType); + return expectedNumberOfElements; } @Test - public void testAppAttempts() throws JSONException, Exception { + public void testAppAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); @@ -1800,7 +1622,8 @@ public void testMultipleAppAttempts() throws JSONException, Exception { int numAttempt = 1; while (true) { // fail the AM by sending CONTAINER_FINISHED event without registering. - amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE); + amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, + ContainerState.COMPLETE); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED); if (numAttempt == maxAppAttempts) { rm.waitForState(app1.getApplicationId(), RMAppState.FAILED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java new file mode 100644 index 00000000000..01abfef69a4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.customresourcetypes.CustomResourceTypesConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.*; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.GuiceServletConfig; +import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.junit.Before; +import org.junit.Test; + +import javax.ws.rs.core.MediaType; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; + +/** + * This test verifies that custom resource types are correctly serialized to XML + * and json when HTTP GET request is sent to the resource: ws/v1/cluster/apps. + */ +public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase { + + private static MockRM rm; + + private static final int CONTAINER_MB = 1024; + + private static class WebServletModule extends ServletModule { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + Configuration conf = new Configuration(); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, + ResourceScheduler.class); + initResourceTypes(conf); + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + serve("/*").with(GuiceContainer.class); + } + + private void initResourceTypes(Configuration conf) { + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + ResourceUtils.resetResourceTypes(conf); + } + } + + static { + createInjectorForWebServletModule(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + createInjectorForWebServletModule(); + } + + private static void createInjectorForWebServletModule() { + GuiceServletConfig + .setInjector(Guice.createInjector(new WebServletModule())); + } + + public TestRMWebServicesAppsCustomResourceTypes() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testRunningAppXml() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); + am1.allocate("*", 2048, 1, new ArrayList<>()); + amNodeManager.nodeHeartbeat(true); + + WebResource r = resource(); + WebResource path = r.path("ws").path("v1").path("cluster").path("apps"); + ClientResponse response = + path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + ArrayWrapper arrayWrapper = responseAdapter.getArray("apps[]"); + assertEquals("incorrect number of elements", 1, arrayWrapper.length()); + + ArrayWrapper appArray = responseAdapter.getArray("apps.app[]"); + assertEquals("incorrect number of elements", 1, appArray.length()); + + verifyAppsXML(arrayWrapper, app1); + }); + + rm.stop(); + } + + @Test + public void testRunningAppJson() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); + am1.allocate("*", 2048, 1, new ArrayList<>()); + amNodeManager.nodeHeartbeat(true); + + WebResource r = resource(); + WebResource path = r.path("ws").path("v1").path("cluster").path("apps"); + ClientResponse response = + path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(path, + new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + assertEquals("incorrect number of elements", 1, + responseAdapter.getElement("apps").length()); + ArrayWrapper appArray = responseAdapter.getArray("apps.app[]"); + assertEquals("incorrect number of elements", 1, appArray.length()); + + verifyAppInfoJson(appArray.getObjectAtIndex(0), app1); + }); + + rm.stop(); + } + + private void verifyAppsXML(ArrayWrapper array, RMApp app) { + for (int i = 0; i < array.length(); i++) { + ElementWrapper element = array.getObjectAtIndex(i); + AppInfoVerifications.verify(element, app); + + ArrayWrapper resourceRequests = element.getChildArray("resourceRequests"); + assertEquals(resourceRequests.length(), 1); + ElementWrapper resourceRequest = resourceRequests.getObjectAtIndex(0); + ResourceRequest rr = + ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) + .getApplicationAttempt( + app.getCurrentAppAttempt().getAppAttemptId()) + .getAppSchedulingInfo().getAllResourceRequests().get(0); + ResourceRequestsVerifications.verifyWithCustomResourceTypes( + resourceRequest, rr, + CustomResourceTypesConfigurationProvider.getCustomResourceTypes()); + } + } + + private void verifyAppInfoJson(ElementWrapper info, RMApp app) { + int expectedNumberOfElements = getExpectedNumberOfElements(app); + + assertEquals("incorrect number of elements", expectedNumberOfElements, + info.length()); + + AppInfoVerifications.verify(info, app); + + ArrayWrapper resourceRequests = info.getChildArray("resourceRequests"); + ElementWrapper requestInfo = resourceRequests.getObjectAtIndex(0); + ResourceRequest rr = + ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) + .getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId()) + .getAppSchedulingInfo().getAllResourceRequests().get(0); + + ResourceRequestsVerifications.verifyWithCustomResourceTypes( + requestInfo, rr, + CustomResourceTypesConfigurationProvider.getCustomResourceTypes()); + } + + private int getExpectedNumberOfElements(RMApp app) { + int expectedNumberOfElements = 38 + 2; // 2 -> resourceRequests + if (app.getApplicationSubmissionContext() + .getNodeLabelExpression() != null) { + expectedNumberOfElements++; + } + + if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { + expectedNumberOfElements++; + } + + if (AppInfo + .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) { + expectedNumberOfElements++; + } + return expectedNumberOfElements; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index edf0652bf35..91ac17b023d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -146,7 +146,7 @@ private static void setupQueueConfiguration( config.setUserLimitFactor(B2, 100.0f); config.setCapacity(B3, 0.5f); config.setUserLimitFactor(B3, 100.0f); - + config.setQueues(A1, new String[] {"a1a", "a1b"}); final String A1A = A1 + ".a1a"; config.setCapacity(A1A, 85); @@ -254,7 +254,7 @@ public void verifyClusterSchedulerXML(NodeList nodes) throws Exception { } } - public void verifySubQueueXML(Element qElem, String q, + public void verifySubQueueXML(Element qElem, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws Exception { NodeList children = qElem.getChildNodes(); @@ -317,26 +317,28 @@ public void verifySubQueueXML(Element qElem, String q, private void verifyClusterScheduler(JSONObject json) throws JSONException, Exception { - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("incorrect number of elements in: " + json, 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); - assertEquals("incorrect number of elements", 1, info.length()); + assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 8, info.length()); + assertEquals("incorrect number of elements in: " + info, 8, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), (float) info.getDouble("maxCapacity"), info.getString("queueName")); JSONObject health = info.getJSONObject("health"); assertNotNull(health); - assertEquals("incorrect number of elements", 3, health.length()); + assertEquals("incorrect number of elements in: " + health, 3, + health.length()); JSONArray arr = info.getJSONObject("queues").getJSONArray("queue"); - assertEquals("incorrect number of elements", 2, arr.length()); + assertEquals("incorrect number of elements in: " + arr, 2, arr.length()); // test subqueues for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); - String q = CapacitySchedulerConfiguration.ROOT + "." + obj.getString("queueName"); + String q = CapacitySchedulerConfiguration.ROOT + "." + + obj.getString("queueName"); verifySubQueue(obj, q, 100, 100); } } @@ -351,7 +353,7 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, assertTrue("queueName doesn't match", "root".matches(queueName)); } - private void verifySubQueue(JSONObject info, String q, + private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { int numExpectedElements = 20; @@ -460,7 +462,7 @@ private void verifyLeafQueueGeneric(String q, LeafQueueInfo info) csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f); } - //Return a child Node of node with the tagname or null if none exists + //Return a child Node of node with the tagname or null if none exists private Node getChildNodeByName(Node node, String tagname) { NodeList nodeList = node.getChildNodes(); for (int i=0; i < nodeList.getLength(); ++i) { @@ -510,7 +512,7 @@ public void testPerUserResourcesXML() throws Exception { for (int j=0; j provider + = new CustomServiceIteratorProvider<>(); + + Iterator> classIterator = provider + .createClassIterator(MessageBodyWriter.class, + SERVICE_NAME, getClass().getClassLoader(), false); + Iterator objIterator = provider.createIterator + (MessageBodyWriter.class, "messageBodyWriter", getClass() + .getClassLoader(), false); + + verifyClassIteratorHasCustomProviderAsFirstElement(classIterator); + + while(objIterator.hasNext()) { + LOG.info("ObjectIterator.next: " + objIterator.next()); + } + } + + private void verifyClassIteratorHasCustomProviderAsFirstElement( + Iterator> classIterator) { + int i = 0; + while(classIterator.hasNext()) { + Class clazz = classIterator.next(); + LOG.info("ClassIterator.next: " + clazz); + + if (i == 0) { + assertEquals(JSONRootElementProviderEclipseLink.App.class, clazz); + } + ++i; + } + + assertTrue("Class iterator should return 2 or more elements!", i > 2); + } + + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java similarity index 73% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java index e77785b929e..58c72eee28c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,13 +16,14 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.resourcemanager.webapp; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import javax.ws.rs.core.MediaType; +package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler; +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -30,6 +31,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -38,18 +42,18 @@ import org.codehaus.jettison.json.JSONObject; import org.junit.Before; import org.junit.Test; +import javax.ws.rs.core.MediaType; -import com.google.inject.Guice; -import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +/** + * Tests RM Webservices fair scheduler resources. + */ public class TestRMWebServicesFairScheduler extends JerseyTestBase { private static MockRM rm; private static YarnConfiguration conf; - + private static class WebServletModule extends ServletModule { @Override protected void configureServlets() { @@ -58,7 +62,7 @@ protected void configureServlets() { bind(GenericExceptionHandler.class); conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, - ResourceScheduler.class); + ResourceScheduler.class); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); @@ -66,32 +70,32 @@ protected void configureServlets() { } static { - GuiceServletConfig.setInjector( - Guice.createInjector(new WebServletModule())); + GuiceServletConfig + .setInjector(Guice.createInjector(new WebServletModule())); } @Before @Override public void setUp() throws Exception { super.setUp(); - GuiceServletConfig.setInjector( - Guice.createInjector(new WebServletModule())); + GuiceServletConfig + .setInjector(Guice.createInjector(new WebServletModule())); } public TestRMWebServicesFairScheduler() { super(new WebAppDescriptor.Builder( "org.apache.hadoop.yarn.server.resourcemanager.webapp") - .contextListenerClass(GuiceServletConfig.class) - .filterClass(com.google.inject.servlet.GuiceFilter.class) - .contextPath("jersey-guice-filter").servletPath("/").build()); + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); } - + @Test - public void testClusterScheduler() throws JSONException, Exception { + public void testClusterScheduler() throws JSONException { WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("scheduler").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + ClientResponse response = + r.path("ws").path("v1").path("cluster").path("scheduler") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); @@ -99,52 +103,51 @@ public void testClusterScheduler() throws JSONException, Exception { } @Test - public void testClusterSchedulerSlash() throws JSONException, Exception { + public void testClusterSchedulerSlash() throws JSONException { WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("scheduler/").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + ClientResponse response = + r.path("ws").path("v1").path("cluster").path("scheduler/") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); verifyClusterScheduler(json); } - + @Test - public void testClusterSchedulerWithSubQueues() throws JSONException, - Exception { - FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler(); + public void testClusterSchedulerWithSubQueues() + throws JSONException { + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); QueueManager queueManager = scheduler.getQueueManager(); // create LeafQueue queueManager.getLeafQueue("root.q.subqueue1", true); queueManager.getLeafQueue("root.q.subqueue2", true); WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("scheduler").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + ClientResponse response = + r.path("ws").path("v1").path("cluster").path("scheduler") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); JSONArray subQueueInfo = json.getJSONObject("scheduler") .getJSONObject("schedulerInfo").getJSONObject("rootQueue") - .getJSONObject("childQueues").getJSONArray("queue") - .getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue"); + .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1) + .getJSONObject("childQueues").getJSONArray("queue"); // subQueueInfo is consist of subqueue1 and subqueue2 info assertEquals(2, subQueueInfo.length()); // Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo. try { subQueueInfo.getJSONObject(1).getJSONObject("childQueues"); - fail("FairSchedulerQueueInfo should omit field 'childQueues'" + - "if child queue is empty."); + fail("FairSchedulerQueueInfo should omit field 'childQueues'" + + "if child queue is empty."); } catch (JSONException je) { assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage()); } } - private void verifyClusterScheduler(JSONObject json) throws JSONException, - Exception { + private void verifyClusterScheduler(JSONObject json) throws JSONException { assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/CustomResourceTypesConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/CustomResourceTypesConfigurationProvider.java new file mode 100644 index 00000000000..4be9278db29 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/CustomResourceTypesConfigurationProvider.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.customresourcetypes; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.LocalConfigurationProvider; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toList; + +/** + * This class can generate an XML configuration file of custom resource types. + * See createInitialResourceTypes for the default values. All custom resource + * type is prefixed with CUSTOM_RESOURCE_PREFIX. Please use the + * getConfigurationInputStream method to get an InputStream of the XML. If you + * want to have different number of resources in your tests, please see usages + * of this class in this test class: + * {@link TestRMWebServicesFairSchedulerCustomResourceTypes} + * + */ +public class CustomResourceTypesConfigurationProvider + extends LocalConfigurationProvider { + + private static class CustomResourceTypes { + private int count; + private String xml; + + CustomResourceTypes(String xml, int count) { + this.xml = xml; + this.count = count; + } + + public int getCount() { + return count; + } + + public String getXml() { + return xml; + } + } + + private static final String CUSTOM_RESOURCE_PREFIX = "customResource-"; + + private static CustomResourceTypes customResourceTypes = + createInitialResourceTypes(); + + private static CustomResourceTypes createInitialResourceTypes() { + return createCustomResourceTypes(2); + } + + private static CustomResourceTypes createCustomResourceTypes(int count) { + List resourceTypeNames = generateResourceTypeNames(count); + + List resourceUnitXmlElements = IntStream.range(0, count) + .boxed() + .map(i -> getResourceUnitsXml(resourceTypeNames.get(i))) + .collect(toList()); + + StringBuilder sb = new StringBuilder("\n"); + sb.append(getResourceTypesXml(resourceTypeNames)); + + for (String resourceUnitXml : resourceUnitXmlElements) { + sb.append(resourceUnitXml); + + } + sb.append(""); + + return new CustomResourceTypes(sb.toString(), count); + } + + private static List generateResourceTypeNames(int count) { + return IntStream.range(0, count) + .boxed() + .map(i -> CUSTOM_RESOURCE_PREFIX + i) + .collect(toList()); + } + + private static String getResourceUnitsXml(String resource) { + return "\n" + "yarn.resource-types." + resource + + ".units\n" + "k\n" + "\n"; + } + + private static String getResourceTypesXml(List resources) { + final String resourceTypes = makeCommaSeparatedString(resources); + + return "\n" + "yarn.resource-types\n" + "" + + resourceTypes + "\n" + "\n"; + } + + private static String makeCommaSeparatedString(List resources) { + return resources.stream().collect(Collectors.joining(",")); + } + + @Override + public InputStream getConfigurationInputStream(Configuration bootstrapConf, + String name) throws YarnException, IOException { + if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) { + return new ByteArrayInputStream( + customResourceTypes.getXml().getBytes()); + } else { + return super.getConfigurationInputStream(bootstrapConf, name); + } + } + + public static void reset() { + customResourceTypes = createInitialResourceTypes(); + } + + public static void setNumberOfResourceTypes(int count) { + customResourceTypes = createCustomResourceTypes(count); + } + + public static List getCustomResourceTypes() { + return generateResourceTypeNames(customResourceTypes.getCount()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/FairSchedulerRepresentationVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/FairSchedulerRepresentationVerifications.java new file mode 100644 index 00000000000..b98ef89e265 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/FairSchedulerRepresentationVerifications.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.customresourcetypes; + +import com.google.common.collect.Sets; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; + +import java.util.Set; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * This test helper class is primarily used by {@link TestRMWebServicesFairSchedulerCustomResourceTypes}. + */ +public class FairSchedulerRepresentationVerifications { + + private static final Set SIMPLE_RESOURCE_CATEGORIES = + Sets.newHashSet("minResources", "amUsedResources", "amMaxResources", + "fairResources", "clusterResources", "reservedResources"); + + private static final Set CUSTOM_RESOURCE_CATEGORIES = + Sets.newHashSet("maxResources", "usedResources", "steadyFairResources", + "demandResources"); + + public void verify(ElementWrapper wrapper) { + verifyResourcesContainDefaultResourceTypes(wrapper, + SIMPLE_RESOURCE_CATEGORIES); + verifyResourcesDoNotContainCustomResourceTypes(wrapper, + SIMPLE_RESOURCE_CATEGORIES); + + verifyResourcesContainsAllResourceTypes(wrapper, + CUSTOM_RESOURCE_CATEGORIES); + } + + private void verifyResourcesContainDefaultResourceTypes(ElementWrapper queue, + Set resourceCategories) { + for (String resourceCategory : resourceCategories) { + boolean hasResourceCategory = queue.hasChild(resourceCategory); + assertTrue("Queue " + queue + " does not have resource category key: " + + resourceCategory, hasResourceCategory); + verifyResourceContainsDefaultResourceTypes( + queue.getChild(resourceCategory)); + } + } + + private void verifyResourceContainsDefaultResourceTypes( + ElementWrapper wrapper) { + Object memory = wrapper.opt("memory"); + Object vCores = wrapper.opt("vCores"); + + assertNotNull("Key 'memory' not found in: " + wrapper, memory); + assertNotNull("Key 'vCores' not found in: " + wrapper, vCores); + } + + private void verifyResourcesDoNotContainCustomResourceTypes( + ElementWrapper firstSubQueue, Set resourceCategories) { + for (String resourceCategory : resourceCategories) { + verifyResourceDoesNotContainCustomResourceTypes( + firstSubQueue.getChild(resourceCategory)); + } + } + + private void verifyResourceDoesNotContainCustomResourceTypes( + ElementWrapper wrapper) { + boolean hasCustomResources = wrapper.hasChild("customResources"); + assertFalse("Resource " + wrapper + " should not have customResources key!", + hasCustomResources); + + for (String resourceType : CustomResourceTypesConfigurationProvider + .getCustomResourceTypes()) { + boolean hasResourceType = wrapper.hasChild(resourceType); + assertFalse("Wrapper " + wrapper + + " should not contain custom resource type: " + resourceType, + hasResourceType); + } + } + + private void verifyResourcesContainsAllResourceTypes(ElementWrapper queue, + Set resourceCategories) { + verifyResourcesContainDefaultResourceTypes(queue, resourceCategories); + verifyResourcesContainCustomResourceTypes(queue, resourceCategories); + } + + private void verifyResourcesContainCustomResourceTypes(ElementWrapper queue, + Set resourceCategories) { + for (String resourceCategory : resourceCategories) { + boolean hasResourceCategory = queue.hasChild(resourceCategory); + assertTrue("Queue " + queue + " does not have key for resourceCategory: " + + resourceCategory, hasResourceCategory); + verifyResourceContainsCustomResourceTypes( + queue.getChild(resourceCategory)); + } + } + + private void verifyResourceContainsCustomResourceTypes( + ElementWrapper elementWrapper) { + for (String resourceType : CustomResourceTypesConfigurationProvider + .getCustomResourceTypes()) { + boolean hasResourceType = elementWrapper.hasChild(resourceType); + assertTrue( + "ElementWrapper " + elementWrapper + + " does not have expected resource type: " + resourceType, + hasResourceType); + Long resourceTypeValue = elementWrapper.getLong(resourceType); + assertNotNull(resourceTypeValue); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/TestRMWebServicesFairSchedulerCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/TestRMWebServicesFairSchedulerCustomResourceTypes.java new file mode 100644 index 00000000000..a01ef3da6f7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/TestRMWebServicesFairSchedulerCustomResourceTypes.java @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.customresourcetypes; + +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices; +import org.apache.hadoop.yarn.server.resourcemanager.webapp + .representationhelper.*; + +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.GuiceServletConfig; +import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import javax.ws.rs.core.MediaType; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +/** + * This class is to test response representations of queue resources, explicitly + * setting custom resource types. with the help of + * {@link CustomResourceTypesConfigurationProvider} + */ +public class TestRMWebServicesFairSchedulerCustomResourceTypes + extends JerseyTestBase { + private static MockRM rm; + private static YarnConfiguration conf; + + private static class WebServletModule extends ServletModule { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, + ResourceScheduler.class); + initResourceTypes(conf); + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + serve("/*").with(GuiceContainer.class); + } + + private void initResourceTypes(YarnConfiguration conf) { + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + ResourceUtils.resetResourceTypes(conf); + } + } + + private static class JsonVerifier implements Consumer { + + @Override + public void accept(ResponseAdapter responseAdapter) { + ArrayWrapper arrayWrapper = responseAdapter.getArray( + "scheduler.schedulerInfo.rootQueue.childQueues" + + ".queue[1].childQueues.queue[]"); + + // childQueueInfo is consist of subqueue1 and subqueue2 info + assertEquals(2, arrayWrapper.length()); + + ElementWrapper firstChildQueue = arrayWrapper.getObjectAtIndex(0); + new FairSchedulerRepresentationVerifications().verify(firstChildQueue); + } + } + + private static class XmlVerifier implements Consumer { + + @Override + public void accept(ResponseAdapter responseAdapter) { + ArrayWrapper arrayWrapper = responseAdapter.getArray( + "scheduler.schedulerInfo.rootQueue.childQueues" + + ".queue[1].childQueues.queue[]"); + assertEquals(2, arrayWrapper.length()); + + ElementWrapper firstChildQueue = arrayWrapper.getObjectAtIndex(0); + new FairSchedulerRepresentationVerifications().verify(firstChildQueue); + } + } + + static { + createInjectorForWebServletModule(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + createInjectorForWebServletModule(); + } + + private static void createInjectorForWebServletModule() { + GuiceServletConfig + .setInjector(Guice.createInjector(new WebServletModule())); + } + + @After + public void teardown() { + CustomResourceTypesConfigurationProvider.reset(); + } + + public TestRMWebServicesFairSchedulerCustomResourceTypes() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testClusterSchedulerWithCustomResourceTypesJson() { + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + QueueManager queueManager = scheduler.getQueueManager(); + // create LeafQueues + queueManager.getLeafQueue("root.q.subqueue1", true); + queueManager.getLeafQueue("root.q.subqueue2", true); + + FSLeafQueue subqueue1 = + queueManager.getLeafQueue("root.q.subqueue1", false); + incrementUsedResourcesOnQueue(subqueue1, 33L); + + WebResource path = + resource().path("ws").path("v1").path("cluster").path("scheduler"); + ClientResponse response = + path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(path, + new BufferedClientResponse(response)); + testCase.verify(new JsonVerifier()); + + ResourceUtils.resetResourceTypes(new Configuration()); + } + + @Test + public void testClusterSchedulerWithCustomResourceTypesXml() { + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + QueueManager queueManager = scheduler.getQueueManager(); + // create LeafQueues + queueManager.getLeafQueue("root.q.subqueue1", true); + queueManager.getLeafQueue("root.q.subqueue2", true); + + FSLeafQueue subqueue1 = + queueManager.getLeafQueue("root.q.subqueue1", false); + incrementUsedResourcesOnQueue(subqueue1, 33L); + + WebResource path = + resource().path("ws").path("v1").path("cluster").path("scheduler"); + ClientResponse response = + path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(new XmlVerifier()); + + ResourceUtils.resetResourceTypes(new Configuration()); + } + + @Test + public void testClusterSchedulerWithElevenCustomResourceTypesXml() { + CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11); + createInjectorForWebServletModule(); + + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + QueueManager queueManager = scheduler.getQueueManager(); + // create LeafQueues + queueManager.getLeafQueue("root.q.subqueue1", true); + queueManager.getLeafQueue("root.q.subqueue2", true); + + FSLeafQueue subqueue1 = + queueManager.getLeafQueue("root.q.subqueue1", false); + incrementUsedResourcesOnQueue(subqueue1, 33L); + + WebResource path = + resource().path("ws").path("v1").path("cluster").path("scheduler"); + ClientResponse response = + path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(new XmlVerifier()); + + ResourceUtils.resetResourceTypes(new Configuration()); + } + + @Test + public void testClusterSchedulerElevenWithCustomResourceTypesJson() { + CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11); + createInjectorForWebServletModule(); + + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + QueueManager queueManager = scheduler.getQueueManager(); + // create LeafQueues + queueManager.getLeafQueue("root.q.subqueue1", true); + queueManager.getLeafQueue("root.q.subqueue2", true); + + FSLeafQueue subqueue1 = + queueManager.getLeafQueue("root.q.subqueue1", false); + incrementUsedResourcesOnQueue(subqueue1, 33L); + + WebResource path = + resource().path("ws").path("v1").path("cluster").path("scheduler"); + ClientResponse response = + path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(path, + new BufferedClientResponse(response)); + testCase.verify(new JsonVerifier()); + + ResourceUtils.resetResourceTypes(new Configuration()); + } + + private void incrementUsedResourcesOnQueue(final FSLeafQueue queue, + final long value) { + try { + Method incUsedResourceMethod = queue.getClass().getSuperclass() + .getDeclaredMethod("incUsedResource", Resource.class); + incUsedResourceMethod.setAccessible(true); + + Map customResources = + CustomResourceTypesConfigurationProvider.getCustomResourceTypes() + .stream() + .collect(Collectors.toMap(Function.identity(), v -> value)); + + incUsedResourceMethod.invoke(queue, + Resource.newInstance(20, 30, customResources)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ArrayWrapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ArrayWrapper.java new file mode 100644 index 00000000000..d4f53d54aaf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ArrayWrapper.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +/** + * An interface class that is intented to be extended by representational + * classes of resources. Currently there are two implementations: + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml.XmlNodeListWrapper} + * and + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json.JsonArrayWrapper} + */ +public interface ArrayWrapper extends Wrapper { + int length(); + ElementWrapper getObjectAtIndex(int idx); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/BufferedClientResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/BufferedClientResponse.java new file mode 100644 index 00000000000..3f8c1665ac9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/BufferedClientResponse.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + + +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.UniformInterfaceException; + +import javax.ws.rs.core.MediaType; +import java.io.IOException; + +/** + * This class is merely a wrapper for {@link ClientResponse}. Given that the + * entity input stream of {@link ClientResponse} can be read only once by + * default and for some tests it is convenient to read the input stream many + * times, this class hides the details of how to do that and prevents + * unnecessary code duplication in tests. + */ +public class BufferedClientResponse { + private ClientResponse response; + + public BufferedClientResponse(ClientResponse response) { + response.bufferEntity(); + this.response = response; + } + + public T getEntity(Class clazz) + throws ClientHandlerException, UniformInterfaceException { + try { + response.getEntityInputStream().reset(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return response.getEntity(clazz); + } + + public MediaType getType() { + return response.getType(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ElementWrapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ElementWrapper.java new file mode 100644 index 00000000000..6431ceff034 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ElementWrapper.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +/** + * This interface is intended to be extended by implementations of + * representational classes. Currently there are two concrete implementations, + * see: + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json.JsonObjectWrapper} + * and + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml.XmlNodeWrapper} + */ +public interface ElementWrapper extends Wrapper { + Object opt(String child); + + Integer getInt(String key); + + Long getLong(String key); + + Float getFloat(String key); + + Double getDouble(String key); + + String getString(String key); + + String getStringSafely(String key); + + Boolean getBoolean(String key); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/JsonCustomResourceTypeTestcase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/JsonCustomResourceTypeTestcase.java new file mode 100644 index 00000000000..62890b0add5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/JsonCustomResourceTypeTestcase.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +import com.sun.jersey.api.client.WebResource; +import org.apache.hadoop.http.JettyUtils; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json.JsonResponseAdapter; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.MediaType; + +import java.util.function.Consumer; + +import static org.junit.Assert.*; + +/** + * This class hides the implementation details of how to verify the structure of + * JSON responses. Tests should only provide the path of the + * {@link WebResource}, the response from the resource and last but not least, + * the verifier Consumer to + * {@link JsonCustomResourceTypeTestcase#verify(Consumer)}. An instance of + * {@link JsonResponseAdapter} will be passed to that consumer to be able to + * verify the response. + */ +public class JsonCustomResourceTypeTestcase { + private static final Logger LOG = + LoggerFactory.getLogger(JsonCustomResourceTypeTestcase.class); + + private final WebResource path; + private final BufferedClientResponse response; + private final JSONObject parsedResponse; + + public JsonCustomResourceTypeTestcase(WebResource path, + BufferedClientResponse response) { + this.path = path; + this.response = response; + this.parsedResponse = response.getEntity(JSONObject.class); + } + + public void verify(Consumer verifier) { + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + logResponse(); + + String responseStr = response.getEntity(String.class); + if (responseStr == null || responseStr.isEmpty()) { + throw new IllegalStateException("Response is null or empty!"); + } + ResponseAdapter responseAdapter = new JsonResponseAdapter(response); + verifier.accept(responseAdapter); + } + + private void logResponse() { + LOG.info("Response from service URL {}: {}", + path.toString(), parsedResponse); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ResponseAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ResponseAdapter.java new file mode 100644 index 00000000000..41793bee895 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ResponseAdapter.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path.ArrayElementPathSegment; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path.Path; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path.PathSegment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertTrue; + +/** + * This class is intended to abstract away the semantics of JSON / XML parser + * libraries, this is done by wrapping the normal elements + * ({@link org.codehaus.jettison.json.JSONObject} / {@link org.w3c.dom.Element} + * and array-typed elements {@link org.codehaus.jettison.json.JSONArray} / + * {@link org.w3c.dom.NodeList}. + * + * The two public methods + * {@link ResponseAdapter#getElement(String)} and + * {@link ResponseAdapter#getArray(String)} are capable of parsing a path that + * describes the hierarchy of the JSON / XML elements. + * + * Example of a path: + * "scheduler.schedulerInfo.rootQueue.childQueues.queue[1].childQueues.queue[]" + * which is the equivalent of this expression with Jettison: + * + *

+ * JSONArray childQueueInfo =
+ *     json.getJSONObject("scheduler")
+ *     .getJSONObject("schedulerInfo")
+ *     .getJSONObject("rootQueue").getJSONObject("childQueues")
+ *     // get the first queue from the array
+ *     .getJSONArray("queue").getJSONObject(1).getJSONObject("childQueues")
+ *     .getJSONArray("queue");
+ * 
+ * + * What this class gives more than the expression above is that + * it checks whether the child element names are found for every parent + * and it also checks that the lengths of the specified arrays contains + * at least as many elements as the queried array element index + 1. + * + * Please note that indexing of the arrays is zero based (developer friendly). + */ +public abstract class ResponseAdapter { + private static final Logger LOG = + LoggerFactory.getLogger(ResponseAdapter.class); + + public ElementWrapper getElement(String pathStr) { + final Path path = Path.create(pathStr); + if (path.isLastSegmentAnArray()) { + throw new IllegalStateException( + "Last segment of part is an array, getArray() should be invoked " + + "instead of this method!"); + } + return (ElementWrapper) getElementInternal(createWrapper(), path); + } + + public ArrayWrapper getArray(String pathStr) { + final Path path = Path.create(pathStr); + if (!path.isLastSegmentAnArray()) { + throw new IllegalStateException( + "Last segment of part is a simple element, getElement() " + + "should be invoked instead of this method!"); + } + return (ArrayWrapper) getElementInternal(createWrapper(), path); + } + + private Wrapper getElementInternal(Wrapper root, Path path) { + LOG.debug("Path processed so far: '{}'", path.getProcessedPath()); + + final PathSegment pathSegment = path.getNextSegment(); + final String child = pathSegment.getSegmentName(); + + final String processedPath = path.getProcessedPath(); + LOG.debug("Processing element: {}", processedPath); + + final Wrapper wrapper; + if (pathSegment.isArrayElementType()) { + checkChildFound(root, child, processedPath); + + final ArrayWrapper array = root.getChildArray(child); + final int elementIndex = ((ArrayElementPathSegment)pathSegment) + .getArrayElementIndex(); + checkArrayIndexIsInBounds(array, elementIndex, processedPath); + + wrapper = array.getObjectAtIndex(elementIndex); + } else if (pathSegment.isArrayType()) { + checkChildFound(root, child, processedPath); + wrapper = root.getChildArray(child); + } else { + checkChildFound(root, child, processedPath); + wrapper = root.getChild(child); + } + + if (!path.isLastSegment()) { + return getElementInternal(wrapper, path); + } else { + return wrapper; + } + } + + private void checkArrayIndexIsInBounds(ArrayWrapper array, + int arrayElementIndex, String processedPath) { + final String message = + String.format( + "Array element %s does not have expected number of elements: %d" + + " at path: %s", + array, (arrayElementIndex + 1), processedPath); + assertTrue(message, array.length() >= (arrayElementIndex + 1)); + } + + private void checkChildFound(Wrapper root, String child, + String processedPath) { + final String message = + String.format("Element %s does not have child element: %s at path: %s", + root, child, processedPath); + assertTrue(message, root.hasChild(child)); + } + + public abstract Wrapper createWrapper(); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/Wrapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/Wrapper.java new file mode 100644 index 00000000000..58416334312 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/Wrapper.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +/** + * This interface is intended to be extended by representational classes. These + * methods are common for JSON / XML representations and also common for simple + * and array elements. + */ +public interface Wrapper { + boolean hasChild(String child); + ElementWrapper getChild(String child); + ArrayWrapper getChildArray(String child); + int length(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/XmlCustomResourceTypeTestCase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/XmlCustomResourceTypeTestCase.java new file mode 100644 index 00000000000..8f20f60260d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/XmlCustomResourceTypeTestCase.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +import com.sun.jersey.api.client.WebResource; +import org.apache.hadoop.http.JettyUtils; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml.XmlResponseAdapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; + +import javax.ws.rs.core.MediaType; +import javax.xml.transform.*; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.StringWriter; +import java.util.function.Consumer; + +import static org.junit.Assert.assertEquals; + +/** + * This class hides the implementation details of how to verify the structure of + * XML responses. Tests should only provide the path of the + * {@link WebResource}, the response from the resource and last but not least, + * the verifier Consumer to + * {@link XmlCustomResourceTypeTestCase#verify(Consumer)}. An instance of + * {@link XmlResponseAdapter} will be passed to that consumer to be able to + * verify the response. + */ +public class XmlCustomResourceTypeTestCase { + private static final Logger LOG = + LoggerFactory.getLogger(XmlCustomResourceTypeTestCase.class); + + private WebResource path; + private BufferedClientResponse response; + + public XmlCustomResourceTypeTestCase(WebResource path, + BufferedClientResponse response) { + this.path = path; + this.response = response; + } + + public void verify(Consumer verifier) { + assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + XmlResponseAdapter responseAdapter = new XmlResponseAdapter(response); + logResponse(responseAdapter.getParsedResponse()); + verifier.accept(responseAdapter); + } + + private void logResponse(Document doc) { + LOG.info("Response from service URL {}: {}", path.toString(), toXml(doc)); + } + + private String toXml(Document doc) { + StringWriter writer; + try { + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty( + "{http://xml.apache.org/xslt}indent" + "-amount", "2"); + writer = new StringWriter(); + transformer.transform(new DOMSource(doc), new StreamResult(writer)); + } catch (TransformerException e) { + throw new RuntimeException(e); + } + + return writer.getBuffer().toString(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonArrayWrapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonArrayWrapper.java new file mode 100644 index 00000000000..bb4eee05c54 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonArrayWrapper.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ArrayWrapper; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; + +/** + * A wrapper for a JSON array. + */ +public class JsonArrayWrapper implements ArrayWrapper { + private JSONArray jsonArray; + + JsonArrayWrapper(JSONArray jsonArray) { + this.jsonArray = jsonArray; + } + + @Override + public int length() { + return jsonArray.length(); + } + + @Override + public ElementWrapper getObjectAtIndex(int idx) { + try { + return new JsonObjectWrapper(jsonArray.getJSONObject(idx)); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean hasChild(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public ElementWrapper getChild(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public ArrayWrapper getChildArray(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return jsonArray.toString(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonObjectWrapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonObjectWrapper.java new file mode 100644 index 00000000000..189c0e2df74 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonObjectWrapper.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ArrayWrapper; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +/** + * A wrapper for a JSON object. + */ +public class JsonObjectWrapper implements ElementWrapper { + private JSONObject jsonObject; + + public JsonObjectWrapper(JSONObject jsonObject) { + this.jsonObject = jsonObject; + } + + @Override + public boolean hasChild(String child) { + return jsonObject.has(child); + } + + @Override + public ElementWrapper getChild(String child) { + try { + return new JsonObjectWrapper(jsonObject.getJSONObject(child)); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public ArrayWrapper getChildArray(String child) { + try { + return new JsonArrayWrapper(jsonObject.getJSONArray(child)); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public int length() { + return jsonObject.length(); + } + + @Override + public Object opt(String child) { + return jsonObject.opt(child); + } + + @Override + public Integer getInt(String key) { + try { + return jsonObject.getInt(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public Float getFloat(String key) { + try { + return (float) jsonObject.getDouble(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public Long getLong(String key) { + try { + return jsonObject.getLong(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public Double getDouble(String key) { + try { + return jsonObject.getDouble(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public String getString(String key) { + try { + return jsonObject.getString(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public String getStringSafely(String key) { + try { + return jsonObject.getString(key); + } catch (JSONException e) { + return null; + } + } + + @Override + public Boolean getBoolean(String key) { + try { + return jsonObject.getBoolean(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString() { + if (jsonObject != null) { + return jsonObject.toString(); + } + return null; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonResponseAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonResponseAdapter.java new file mode 100644 index 00000000000..ea53699f236 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonResponseAdapter.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.BufferedClientResponse; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ResponseAdapter; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.Wrapper; +import org.codehaus.jettison.json.JSONObject; + +/** + * A wrapper for a response that is converted to a {@link JSONObject}. + */ +public class JsonResponseAdapter extends ResponseAdapter { + private JSONObject response; + + public JsonResponseAdapter(BufferedClientResponse response) { + this.response = response.getEntity(JSONObject.class); + } + + @Override + public Wrapper createWrapper() { + return new JsonObjectWrapper(response); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayElementPathSegment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayElementPathSegment.java new file mode 100644 index 00000000000..475725b9d35 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayElementPathSegment.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + +import java.util.regex.Matcher; + +/** + * An implementation of {@link PathSegment} that represents a concrete element + * of an array. + */ +public class ArrayElementPathSegment extends PathSegment { + private final int arrayElementIndex; + + ArrayElementPathSegment(String segmentName, String pathPart) { + super(segmentName); + this.arrayElementIndex = extractArrayElementIndex(pathPart); + } + + private int extractArrayElementIndex(String pathPart) { + int arrayElementIndex = extractIndex(pathPart); + if (arrayElementIndex >= 0) { + return arrayElementIndex; + } else { + throw new IllegalStateException( + "Array arrayElementIndex should be greater than 0!"); + } + } + + private int extractIndex(String pathPart) { + final Matcher matcher = ARRAY_ELEMENT_PATTERN.matcher(pathPart); + + // this call is intentional, without this, + // matcher groups were empty even if matched! + matcher.matches(); + try { + return Integer.parseInt(matcher.group(1)); + } catch (NumberFormatException e) { + throw new IllegalStateException( + "Cannot parse array element index: " + matcher.group()); + } + } + + public int getArrayElementIndex() { + return arrayElementIndex; + } + + @Override + PathSegmentType getType() { + return PathSegmentType.ARRAY_ELEMENT; + } + + @Override + public String toString() { + return String.format("%s[%d]", getSegmentName(), arrayElementIndex); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayPathSegment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayPathSegment.java new file mode 100644 index 00000000000..b7c5bd0e395 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayPathSegment.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + + +/** + * An implementation of {@link PathSegment} that represents an array without its + * concrete element. + */ +public class ArrayPathSegment extends PathSegment { + ArrayPathSegment(String segmentName) { + super(segmentName); + } + + @Override + PathSegmentType getType() { + return PathSegmentType.ARRAY; + } + + @Override + public String toString() { + return getSegmentName() + "[]"; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/NormalPathSegment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/NormalPathSegment.java new file mode 100644 index 00000000000..83f0a2ee989 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/NormalPathSegment.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + +/** + * An implementation of {@link PathSegment} that represents normal element. + */ +public class NormalPathSegment extends PathSegment { + + NormalPathSegment(String segmentName) { + super(segmentName); + } + + @Override + PathSegmentType getType() { + return PathSegmentType.NORMAL; + } + + @Override + public String toString() { + return getSegmentName(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/Path.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/Path.java new file mode 100644 index 00000000000..48e91795de9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/Path.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + + +import com.google.common.collect.Lists; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * This class is a generic definition of an XML / JSON path. See explanation for + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ResponseAdapter} + * for details about why this is useful in tests. A Path is basically a list of + * {@link PathSegment}s. On top of this, this class has helper methods that can + * process the path sequentially, and always knows what is the current + * {@link PathSegment} which is being processed. + */ +public class Path { + private final List segments; + private int currentIndex; + private final boolean lastSegmentArray; + + public Path(List pathSegments) { + this.segments = pathSegments; + this.currentIndex = 0; + this.lastSegmentArray = + pathSegments.get(pathSegments.size() - 1).isArrayType(); + } + + public static Path create(String path) { + final List pathSegments; + if (path.contains(".")) { + String[] pathParts = path.split("\\."); + + pathSegments = Lists.newArrayList(); + for (String pathPart : pathParts) { + pathSegments.add(PathSegment.create(pathPart)); + } + } else { + pathSegments = Lists.newArrayList(PathSegment.create(path)); + } + return new Path(pathSegments); + } + + public PathSegment getNextSegment() { + return segments.get(currentIndex++); + } + + public boolean isLastSegment() { + return segments.size() == currentIndex; + } + + public boolean isLastSegmentAnArray() { + return lastSegmentArray; + } + + public String getProcessedPath() { + final List subList = segments.subList(0, currentIndex); + return subList.stream().map(PathSegment::toString) + .collect(Collectors.joining(".")); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegment.java new file mode 100644 index 00000000000..610f766db35 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegment.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A segment of a path. + * There are three types of segments, see implementations of {@link PathSegment}. + * The type is determined by the string representation of segments, for example: + * array type is matches for the ARRAY_PATTERN. + */ +public abstract class PathSegment { + + private static final Pattern ARRAY_PATTERN = Pattern.compile(".*\\[\\]"); + static final Pattern ARRAY_ELEMENT_PATTERN = + Pattern.compile(".*\\[(\\d+)\\]"); + + private final String segmentName; + + PathSegment(String segmentName) { + this.segmentName = segmentName; + } + + static PathSegment create(final String pathPart) { + final PathSegmentType type = determineType(pathPart); + final String segmentName = extractSegmentName(type, pathPart); + + if (type == PathSegmentType.ARRAY_ELEMENT) { + return new ArrayElementPathSegment(segmentName, pathPart); + } else if (type == PathSegmentType.ARRAY) { + return new ArrayPathSegment(segmentName); + } else { + return new NormalPathSegment(segmentName); + } + } + + private static PathSegmentType determineType(String pathPart) { + final boolean arrayType = ARRAY_PATTERN.matcher(pathPart).matches(); + final Matcher arrayElementMatcher = + ARRAY_ELEMENT_PATTERN.matcher(pathPart); + + if (arrayElementMatcher.matches()) { + return PathSegmentType.ARRAY_ELEMENT; + } else if (arrayType) { + return PathSegmentType.ARRAY; + } else { + return PathSegmentType.NORMAL; + } + } + + private static String extractSegmentName( + PathSegmentType type, String pathPart) { + if (type == PathSegmentType.ARRAY_ELEMENT || + type == PathSegmentType.ARRAY) { + return extractSegmentName(pathPart); + } else { + return pathPart; + } + } + + private static String extractSegmentName(String pathPart) { + return pathPart.substring(0, pathPart.indexOf('[')); + } + + abstract PathSegmentType getType(); + + public String getSegmentName() { + return segmentName; + } + + public boolean isArrayType() { + return getType() == PathSegmentType.ARRAY; + } + + public boolean isArrayElementType() { + return getType() == PathSegmentType.ARRAY_ELEMENT; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegmentType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegmentType.java new file mode 100644 index 00000000000..08392382e23 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegmentType.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + +public enum PathSegmentType { + NORMAL, ARRAY, ARRAY_ELEMENT +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeListWrapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeListWrapper.java new file mode 100644 index 00000000000..de8f97bbd35 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeListWrapper.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml; + + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ArrayWrapper; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; +import org.w3c.dom.NodeList; + +/** + * A wrapper for an XML nodelist. + */ +public class XmlNodeListWrapper implements ArrayWrapper{ + private NodeList nodes; + + XmlNodeListWrapper(NodeList nodes) { + this.nodes = nodes; + } + + @Override + public int length() { + return nodes.getLength(); + } + + @Override + public ElementWrapper getObjectAtIndex(int idx) { + return new XmlNodeWrapper(nodes.item(idx)); + } + + @Override + public boolean hasChild(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public ElementWrapper getChild(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public ArrayWrapper getChildArray(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return nodes.toString(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeWrapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeWrapper.java new file mode 100644 index 00000000000..882fe1b4af7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeWrapper.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ArrayWrapper; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * A wrapper class for an XML node. + */ +public class XmlNodeWrapper implements ElementWrapper { + private Node node; + + XmlNodeWrapper(Node node) { + this.node = node; + } + + @Override + public boolean hasChild(String child) { + return getElementsByTagNameInternal(child).getLength() > 0; + } + + @Override + public ElementWrapper getChild(String child) { + Node item = getElementsByTagNameInternal(child).item(0); + return new XmlNodeWrapper(item); + } + + @Override + public ArrayWrapper getChildArray(String child) { + NodeList nodes = getElementsByTagNameInternal(child); + return new XmlNodeListWrapper(nodes); + } + + @Override + public int length() { + // node.getChildNodes().getLength() would return textNodes as well, + // we only need Elements + int count = 0; + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node current = children.item(i); + if (current.getNodeType() == Node.ELEMENT_NODE) { + ++count; + } + } + + return count; + } + + @Override + public Object opt(String child) { + NodeList nodes = getElementsByTagNameInternal(child); + if (nodes.getLength() > 0) { + return nodes.item(0); + } + + return null; + } + + @Override + public Integer getInt(String key) { + return WebServicesTestUtils.getXmlInt((Element) node, key); + } + + @Override + public Float getFloat(String key) { + return WebServicesTestUtils.getXmlFloat((Element) node, key); + } + + @Override + public Long getLong(String key) { + return WebServicesTestUtils.getXmlLong((Element) node, key); + } + + @Override + public Double getDouble(String key) { + return (double) WebServicesTestUtils.getXmlFloat((Element) node, key); + } + + @Override + public String getString(String key) { + return WebServicesTestUtils.getXmlString((Element) node, key); + } + + @Override + public String getStringSafely(String key) { + return getString(key); + } + + @Override + public Boolean getBoolean(String key) { + return WebServicesTestUtils.getXmlBoolean((Element) node, key); + } + + @Override + public String toString() { + if (node != null) { + return node.toString(); + } + return null; + } + + private NodeList getElementsByTagNameInternal(String child) { + if (node instanceof Element) { + return ((Element) node).getElementsByTagName(child); + } else if (node instanceof Document) { + return ((Document) node).getElementsByTagName(child); + } else { + throw new IllegalStateException("Unknown type of wrappedObject: " + + node + ", type: " + node.getClass()); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlResponseAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlResponseAdapter.java new file mode 100644 index 00000000000..52706660ca9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlResponseAdapter.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml; + + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.BufferedClientResponse; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ResponseAdapter; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.Wrapper; + +import org.w3c.dom.Document; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.StringReader; + +/** + * A wrapper for a response that is converted to a {@link Document}. + */ +public class XmlResponseAdapter extends ResponseAdapter { + private final Document parsedResponse; + + public XmlResponseAdapter(BufferedClientResponse response) { + this.parsedResponse = parseXml(response); + } + + private Document parseXml(BufferedClientResponse response) { + try { + String xml = response.getEntity(String.class); + DocumentBuilder db = DocumentBuilderFactory.newInstance() + .newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + + return db.parse(is); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public Document getParsedResponse() { + return parsedResponse; + } + + @Override + public Wrapper createWrapper() { + return new XmlNodeWrapper(parsedResponse); + } +}