Index: hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml (date 1515392946000) +++ hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml (date 1513958977000) @@ -84,7 +84,7 @@ but enforcer still sees it. --> org.apache.hadoop:hadoop-annotations - + org.apache.htrace:htrace-core4 org.slf4j:slf4j-api Index: hadoop-client-modules/hadoop-client-minicluster/pom.xml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-client-modules/hadoop-client-minicluster/pom.xml (date 1515392946000) +++ hadoop-client-modules/hadoop-client-minicluster/pom.xml (date 1513958977000) @@ -571,7 +571,7 @@ org.apache.hadoop:hadoop-yarn-server-timelineservice log4j:log4j - + @@ -662,6 +662,27 @@ capacity-scheduler.xml krb5.conf + + + + org.eclipse.persistence:eclipselink + + **/*.html + + + + org.eclipse.persistence:javax.persistence + + **/*.html + + + + org.eclipse.persistence:commonj.sdo + + **/*.html + about_files/** + xsd/** + plugin.properties @@ -778,6 +799,35 @@ javax/ws/ ${shaded.dependency.prefix}.javax.ws. + + **/pom.xml + + + + + javax/persistence/ + ${shaded.dependency.prefix}.javax.persistence. + + **/pom.xml + + + + javax/validation/ + ${shaded.dependency.prefix}.javax.validation. + + **/pom.xml + + + + commonj/ + ${shaded.dependency.prefix}.commonj. + + **/pom.xml + + + + javax/json/ + ${shaded.dependency.prefix}.javax.json. **/pom.xml Index: hadoop-project/pom.xml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-project/pom.xml (date 1515392946000) +++ hadoop-project/pom.xml (date 1513958977000) @@ -146,6 +146,7 @@ 1.5.4 1.16 + 2.6.0 @@ -1356,6 +1357,11 @@ assertj-core 3.8.0 test + + + org.eclipse.persistence + eclipselink + ${eclipselink.version} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (date 1513958977000) @@ -250,6 +250,11 @@ jersey-test-framework-grizzly2 test + + + org.eclipse.persistence + eclipselink + Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (date 1513958977000) @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import com.google.common.annotations.VisibleForTesting; +import com.sun.jersey.spi.service.ServiceFinder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.curator.framework.AuthInfo; @@ -268,7 +269,7 @@ loadConfigurationXml(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); validateConfigs(this.conf); - + // Set HA configuration should be done before login this.rmContext.setHAEnabled(HAUtil.isHAEnabled(this.conf)); if (this.rmContext.isHAEnabled()) { @@ -312,7 +313,7 @@ } rmContext.setYarnConfiguration(conf); - + createAndInitActiveServices(false); webAppAddress = WebAppUtils.getWebAppBindURL(this.conf, @@ -485,12 +486,12 @@ protected AMLivelinessMonitor createAMLivelinessMonitor() { return new AMLivelinessMonitor(this.rmDispatcher); } - + protected RMNodeLabelsManager createNodeLabelManager() throws InstantiationException, IllegalAccessException { return new RMNodeLabelsManager(); } - + protected DelegationTokenRenewer createDelegationTokenRenewer() { return new DelegationTokenRenewer(); } @@ -599,7 +600,7 @@ AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor(); addService(amFinishingMonitor); rmContext.setAMFinishingMonitor(amFinishingMonitor); - + RMAppLifetimeMonitor rmAppLifetimeMonitor = createRMAppLifetimeMonitor(); addService(rmAppLifetimeMonitor); rmContext.setRMAppLifetimeMonitor(rmAppLifetimeMonitor); @@ -1041,7 +1042,7 @@ RMWebAppUtil.setupSecurityAndFilters(conf, getClientRMService().rmDTSecretManager); - Builder builder = + Builder builder = WebApps .$for("cluster", ApplicationMasterService.class, masterService, "ws") @@ -1109,6 +1110,11 @@ serviceConfig.put("PackageName", apiPackages); serviceConfig.put("PathSpec", "/app/*"); } + + ServiceFinder.setIteratorProvider(new org.apache.hadoop.yarn.server + .resourcemanager.webapp.dao.helper.ServiceFinder.DefaultServiceIteratorProvider()); + + webApp = builder.start(new RMWebApp(this), uiWebAppContext, serviceConfig); } @@ -1227,7 +1233,7 @@ } super.serviceStart(); } - + protected void doSecureLogin() throws IOException { InetSocketAddress socAddr = getBindAddress(conf); SecurityUtil.login(this.conf, YarnConfiguration.RM_KEYTAB, @@ -1257,7 +1263,7 @@ transitionToStandby(false); rmContext.setHAServiceState(HAServiceState.STOPPING); } - + protected ResourceTrackerService createResourceTrackerService() { return new ResourceTrackerService(this.rmContext, this.nodesListManager, this.nmLivelinessMonitor, @@ -1465,7 +1471,7 @@ /** * Retrieve RM bind address from configuration - * + * * @param conf * @return InetSocketAddress */ Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/Application.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/Application.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/Application.java (date 1513958977000) @@ -0,0 +1,11 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import com.sun.jersey.api.core.DefaultResourceConfig; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.JSONRootElementProviderEclipseLink; + + +public class Application extends DefaultResourceConfig { + public Application() { + super(JSONRootElementProviderEclipseLink.class); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java (date 1513958977000) @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import com.google.common.collect.Maps; import com.google.inject.Singleton; import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONJAXBContext; @@ -27,55 +28,118 @@ import javax.ws.rs.ext.ContextResolver; import javax.ws.rs.ext.Provider; import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*; import org.apache.hadoop.yarn.webapp.RemoteExceptionData; +import org.eclipse.persistence.jaxb.JAXBContextFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Singleton @Provider public class JAXBContextResolver implements ContextResolver { + private static final Logger LOG = + LoggerFactory.getLogger(JAXBContextResolver.class); + static final JSONConfiguration JSON_CONFIG_DEFAULT = + JSONConfiguration.natural().rootUnwrapping(false).build(); + + private static final JSONConfiguration JSON_CONFIG_UNWRAPPED = + JSONConfiguration.natural().rootUnwrapping(true).build(); private final Map typesContextMap; - public JAXBContextResolver() throws Exception { - - JAXBContext context; - JAXBContext unWrappedRootContext; - - // you have to specify all the dao classes here - final Class[] cTypes = - { AppInfo.class, AppAttemptInfo.class, AppAttemptsInfo.class, - ClusterInfo.class, CapacitySchedulerQueueInfo.class, - FifoSchedulerInfo.class, SchedulerTypeInfo.class, NodeInfo.class, - UserMetricsInfo.class, CapacitySchedulerInfo.class, - ClusterMetricsInfo.class, SchedulerInfo.class, AppsInfo.class, - NodesInfo.class, RemoteExceptionData.class, - CapacitySchedulerQueueInfoList.class, ResourceInfo.class, - UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, - StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, - FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, - AppTimeoutInfo.class }; - // these dao classes need root unwrapping - final Class[] rootUnwrappedTypes = - { NewApplication.class, ApplicationSubmissionContextInfo.class, - ContainerLaunchContextInfo.class, LocalResourceInfo.class, - DelegationToken.class, AppQueue.class, AppPriority.class }; + /** + * You have to specify all the dao classes here + */ + private static final Class[] NORMAL_TYPES = new Class[] { + AppAttemptInfo.class, AppAttemptsInfo.class, ClusterInfo.class, + CapacitySchedulerQueueInfo.class, FifoSchedulerInfo.class, NodeInfo.class, + UserMetricsInfo.class, CapacitySchedulerInfo.class, + ClusterMetricsInfo.class, SchedulerInfo.class, NodesInfo.class, + RemoteExceptionData.class, CapacitySchedulerQueueInfoList.class, + ResourceInfo.class, UsersInfo.class, UserInfo.class, + ApplicationStatisticsInfo.class, StatisticsItemInfo.class, + CapacitySchedulerHealthInfo.class, FairSchedulerQueueInfoList.class, + AppTimeoutsInfo.class, AppTimeoutInfo.class }; + + /** + * These dao classes need root unwrapping + */ + private static final Class[] ROOT_UNWRAPPED_TYPES = new Class[] { + NewApplication.class, ApplicationSubmissionContextInfo.class, + ContainerLaunchContextInfo.class, LocalResourceInfo.class, + DelegationToken.class, AppQueue.class, AppPriority.class }; - this.typesContextMap = new HashMap(); - context = - new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(false) - .build(), cTypes); - unWrappedRootContext = - new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(true) - .build(), rootUnwrappedTypes); - for (Class type : cTypes) { - typesContextMap.put(type, context); + public static final Class[] ECLIPSELINK_SERIALIZED_TYPES = + new Class[] { SchedulerTypeInfo.class, AppsInfo.class, AppInfo.class, + ResourceInfoWithCustomResourceTypes.class }; + + public JAXBContextResolver() throws Exception { + this.typesContextMap = createTypesContextMap(); + } + + private Map createTypesContextMap() throws Exception { + Map jaxbContexts = Maps.newHashMap(); + jaxbContexts.putAll(getJAXBContextsForNormalClasses()); + jaxbContexts.putAll(getJAXBContextsForUnwrappedClasses()); + jaxbContexts.putAll(getJAXBContextForEclipseLinkClasses()); + + return jaxbContexts; + } + + private Map getJAXBContextsForNormalClasses() + throws JAXBException { + LOG.debug( + "Registering classes with json config without root unwrapping: {}", + Arrays.toString(NORMAL_TYPES)); + Map result = Maps.newHashMap(); + + JAXBContext context = + new JSONJAXBContext(JSON_CONFIG_DEFAULT, NORMAL_TYPES); + for (Class type : NORMAL_TYPES) { + result.put(type, context); } - for (Class type : rootUnwrappedTypes) { - typesContextMap.put(type, unWrappedRootContext); + + return result; + } + + private Map getJAXBContextsForUnwrappedClasses() + throws JAXBException { + LOG.debug("Registering classes with json config with root unwrapping: {}", + Arrays.toString(ROOT_UNWRAPPED_TYPES)); + Map result = Maps.newHashMap(); + + JAXBContext context = + new JSONJAXBContext(JSON_CONFIG_UNWRAPPED, ROOT_UNWRAPPED_TYPES); + for (Class type : ROOT_UNWRAPPED_TYPES) { + result.put(type, context); } + + return result; } + + /** + * Sse Eclipselink JAXBProvider for the specified classes, see + * ResourceInfoWithCustomResourceTypes.customResources as an example, it uses + * EclipseLink JAXBProvider's @XMlPath annotation. + */ + private Map getJAXBContextForEclipseLinkClasses() + throws JAXBException { + LOG.debug("Registering classes with json config EclipseLink marshaller: {}", + Arrays.toString(ECLIPSELINK_SERIALIZED_TYPES)); + Map result = Maps.newHashMap(); + + JAXBContext context = JAXBContextFactory.createContext( + ECLIPSELINK_SERIALIZED_TYPES, Collections. emptyMap()); + + for (Class type : ECLIPSELINK_SERIALIZED_TYPES) { + result.put(type, context); + } + + return result; + } @Override public JAXBContext getContext(Class objectType) { Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java (date 1513958977000) @@ -18,22 +18,21 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; -import static org.apache.hadoop.yarn.util.StringHelper.pajoin; - -import java.net.InetSocketAddress; - +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.util.RMHAUtils; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.util.RMHAUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnWebParams; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import java.net.InetSocketAddress; + +import static org.apache.hadoop.yarn.util.StringHelper.pajoin; /** * The RM webapp Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java (date 1513958977000) @@ -21,11 +21,8 @@ import java.util.List; import java.util.Map; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.*; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -46,11 +43,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields; import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields.DeSelectType; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.MapAdapter; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; +import org.eclipse.persistence.oxm.annotations.XmlPath; @XmlRootElement(name = "app") @XmlAccessorType(XmlAccessType.FIELD) @@ -101,6 +101,11 @@ private long vcoreSeconds; protected float queueUsagePercentage; protected float clusterUsagePercentage; + + // EclipseLink JAXB provider would not include this field if empty without + // these annotations + @XmlJavaTypeAdapter(MapAdapter.class) + @XmlPath("resourceSecondsMap") protected Map resourceSecondsMap; // preemption info fields @@ -110,12 +115,15 @@ private int numAMContainerPreempted; private long preemptedMemorySeconds; private long preemptedVcoreSeconds; + + // EclipseLink JAXB provider would not include this field if empty without + // these annotations + @XmlJavaTypeAdapter(MapAdapter.class) + @XmlPath("preemptedResourceSecondsMap") protected Map preemptedResourceSecondsMap; - // list of resource requests - @XmlElement(name = "resourceRequests") private List resourceRequests = - new ArrayList(); + new ArrayList<>(); protected LogAggregationStatus logAggregationStatus; protected boolean unmanagedApplication; @@ -470,7 +478,7 @@ public int getNumNonAMContainersPreempted() { return numNonAMContainerPreempted; } - + public int getNumAMContainersPreempted() { return numAMContainerPreempted; } @@ -499,6 +507,11 @@ return preemptedResourceSecondsMap; } + public void setPreemptedResourceSecondsMap(Map + preemptedResourceSecondsMap) { + this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; + } + public List getResourceRequests() { return this.resourceRequests; } @@ -615,4 +628,8 @@ public void setName(String name) { this.name = name; } + + public void setResourceSecondsMap(Map resourceSecondsMap) { + this.resourceSecondsMap = resourceSecondsMap; + } } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (date 1513958977000) @@ -51,12 +51,12 @@ private float fractionMemMaxShare; private ResourceInfo minResources; - private ResourceInfo maxResources; - private ResourceInfo usedResources; + private ResourceInfoWithCustomResourceTypes maxResources; + private ResourceInfoWithCustomResourceTypes usedResources; private ResourceInfo amUsedResources; private ResourceInfo amMaxResources; - private ResourceInfo demandResources; - private ResourceInfo steadyFairResources; + private ResourceInfoWithCustomResourceTypes demandResources; + private ResourceInfoWithCustomResourceTypes steadyFairResources; private ResourceInfo fairResources; private ResourceInfo clusterResources; private ResourceInfo reservedResources; @@ -88,15 +88,15 @@ amMaxResources = new ResourceInfo(Resource.newInstance( queue.getMetrics().getMaxAMShareMB(), queue.getMetrics().getMaxAMShareVCores())); - usedResources = new ResourceInfo(queue.getResourceUsage()); - demandResources = new ResourceInfo(queue.getDemand()); + usedResources = new ResourceInfoWithCustomResourceTypes(queue.getResourceUsage()); + demandResources = new ResourceInfoWithCustomResourceTypes(queue.getDemand()); fractionMemUsed = (float)usedResources.getMemorySize() / clusterResources.getMemorySize(); - steadyFairResources = new ResourceInfo(queue.getSteadyFairShare()); + steadyFairResources = new ResourceInfoWithCustomResourceTypes(queue.getSteadyFairShare()); fairResources = new ResourceInfo(queue.getFairShare()); minResources = new ResourceInfo(queue.getMinShare()); - maxResources = new ResourceInfo( + maxResources = new ResourceInfoWithCustomResourceTypes( Resources.componentwiseMin(queue.getMaxShare(), scheduler.getClusterResource())); reservedResources = new ResourceInfo(queue.getReservedResource()); @@ -168,7 +168,7 @@ /** * Returns the steady fair share of this queue in megabytes. */ - public ResourceInfo getSteadyFairShare() { + public ResourceInfoWithCustomResourceTypes getSteadyFairShare() { return steadyFairResources; } @@ -183,7 +183,7 @@ return minResources; } - public ResourceInfo getMaxResources() { + public ResourceInfoWithCustomResourceTypes getMaxResources() { return maxResources; } @@ -199,7 +199,7 @@ return queueName; } - public ResourceInfo getUsedResources() { + public ResourceInfoWithCustomResourceTypes getUsedResources() { return usedResources; } @@ -220,7 +220,7 @@ /** * @return the demand resource of this queue. */ - public ResourceInfo getDemandResources() { + public ResourceInfoWithCustomResourceTypes getDemandResources() { return demandResources; } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfoWithCustomResourceTypes.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfoWithCustomResourceTypes.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfoWithCustomResourceTypes.java (date 1513958977000) @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.MapAdapter; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.eclipse.persistence.oxm.annotations.XmlPath; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import java.util.Map; +import java.util.Set; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.NONE) +public class ResourceInfoWithCustomResourceTypes { + private static final Set DEFAULT_RESOURCE_NAMES = + Sets.newHashSet("memory", "vcores"); + + @XmlElement + long memory; + @XmlElement + int vCores; + + private Resource resources; + + /** + * XmlPath produces a flattened structure, like the keys / values of map were + * fields of this object. Example: + * "maxResources":{"memory":0,"vCores":0,"customResource1":11,"customResource2":22} + */ + @XmlJavaTypeAdapter(MapAdapter.class) + @XmlPath(".") + private Map customResources = Maps.newHashMap(); + + public ResourceInfoWithCustomResourceTypes() { + } + + ResourceInfoWithCustomResourceTypes(final Resource res) { + memory = res.getMemorySize(); + vCores = res.getVirtualCores(); + resources = Resources.clone(res); + customResources = createCustomResources(res); + } + + /** + * + * @param res + * @return + */ + private static Map createCustomResources(final Resource res) { + final ResourceInformation[] resourceInformations = res.getResources(); + + final Map result = Maps.newHashMap(); + for (ResourceInformation ri : resourceInformations) { + final String name = ri.getName(); + + if (!isDefaultResource(name)) { + result.put(name, ri.getValue()); + } + } + + return result; + } + + private static boolean isDefaultResource(final String name) { + final String nameLowerCase = name.toLowerCase(); + + for (String defaultResourceName : DEFAULT_RESOURCE_NAMES) { + if (nameLowerCase.contains(defaultResourceName)) { + return true; + } + } + return false; + } + + public long getMemorySize() { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + return resources.getMemorySize(); + } + + public int getvCores() { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + return resources.getVirtualCores(); + } + + @Override + public String toString() { + return resources.toString(); + } + + public void setMemory(int memory) { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + this.memory = memory; + resources.setMemorySize(memory); + } + + public void setvCores(int vCores) { + if (resources == null) { + resources = Resource.newInstance(memory, vCores); + } + this.vCores = vCores; + resources.setVirtualCores(vCores); + } + + public Resource getResource() { + return Resource.newInstance(resources); + } + + public Map getCustomResources() { + return customResources; + } + + public void setCustomResources(Map customResources) { + this.customResources = customResources; + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java (date 1513958977000) @@ -39,7 +39,7 @@ @XmlElement(name = "resourceName") private String resourceName; @XmlElement(name = "capability") - private ResourceInfo capability; + private ResourceInfoWithCustomResourceTypes capability; @XmlElement(name = "numContainers") private int numContainers; @XmlElement(name = "relaxLocality") @@ -61,7 +61,7 @@ public ResourceRequestInfo(ResourceRequest request) { priority = request.getPriority().getPriority(); resourceName = request.getResourceName(); - capability = new ResourceInfo(request.getCapability()); + capability = new ResourceInfoWithCustomResourceTypes(request.getCapability()); numContainers = request.getNumContainers(); relaxLocality = request.getRelaxLocality(); nodeLabelExpression = request.getNodeLabelExpression(); @@ -87,11 +87,11 @@ this.resourceName = resourceName; } - public ResourceInfo getCapability() { + public ResourceInfoWithCustomResourceTypes getCapability() { return capability; } - public void setCapability(ResourceInfo capability) { + public void setCapability(ResourceInfoWithCustomResourceTypes capability) { this.capability = capability; } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java (date 1513958977000) @@ -23,6 +23,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; +import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -74,9 +75,13 @@ } public String getSchedulerResourceTypes() { - return Arrays.toString(minAllocResource.getResource().getResources()); + if (minAllocResource != null) { + return Arrays.toString(minAllocResource.getResource().getResources()); + } + return null; } + @XmlTransient public int getMaxClusterLevelAppPriority() { return this.maximumClusterPriority; } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/JSONRootElementProviderEclipseLink.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/JSONRootElementProviderEclipseLink.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/JSONRootElementProviderEclipseLink.java (date 1513958977000) @@ -0,0 +1,233 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; + +import com.google.common.collect.ImmutableSet; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; +import com.sun.jersey.core.header.QualitySourceMediaType; +import com.sun.jersey.core.provider.jaxb.AbstractRootElementProvider; +import com.sun.jersey.core.util.FeaturesAndProperties; +import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; +import javax.ws.rs.Consumes; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.ext.Providers; +import javax.xml.bind.*; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Set; + +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver.ECLIPSELINK_SERIALIZED_TYPES; + +/** + * + * This class is an extension of {@link JSONRootElementProvider} in a sense that + * it replaces the functionality of the + * {@link JSONRootElementProvider#writeTo(Object, MediaType, Charset, Marshaller, OutputStream)} + * with a custom implementation. + * + * The reason why the original + * {@link JSONRootElementProvider#writeTo(Object, MediaType, Charset, Marshaller, OutputStream)} + * is not adequate is that it calls + * {@link JSONJAXBContext#getJSONMarshaller(Marshaller, JAXBContext)} that + * creates a {@link com.sun.jersey.json.impl.BaseJSONMarshaller} in case of the + * marshaller is not an instance of {@link JSONMarshaller}. + * + * Currently there are some POJOs, e.g. {@link SchedulerTypeInfo}, + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo} that + * are serialized with EclipseLink's JAXB implementation, see implementation in + * {@link JAXBContextResolver#createTypesContextMap()}. If a class with + * EclipseLink JAXB is being serialized, the marshaller created by Jersey is not + * an instance of {@link JSONMarshaller} therefore + * {@link com.sun.jersey.api.json.JSONConfiguration#DEFAULT} would be used as a + * JSON config for serialization. + * + * Since the {@link com.sun.jersey.api.json.JSONConfiguration#DEFAULT} is a + * hardcoded JSON config and it turns on root unwrapping, therefore the root tag + * would not have been serialized. + * + * A more simplistic approach would have been to replace + * {@link com.sun.jersey.api.json.JSONConfiguration#DEFAULT} with a more + * feasible configuration (without root unwrapping) but it caused several tests + * to fail since some webservice relied on the default configuration. + * + * Every operation except + * {@link JSONRootElementProviderEclipseLink#writeTo(Object, MediaType, Charset, Marshaller, OutputStream)} + * is delegated to {@link JSONRootElementProvider} methods, respectively. + * Subclassing the delegate is not possible as constructors are package-private. + * + * The only special thing is in + * {@link App#isWriteable(Class, Type, Annotation[], MediaType)} that only + * treats a passed-in type as supported if it is included in the + * {@link App#SUPPORTED_TYPES} set. This way, we can limit the serialization + * representation's scope to such supported classes. Delegate method + * {@link JSONRootElementProviderEclipseLink#readFrom(Class, MediaType, Unmarshaller, InputStream)} + * is treated special since this method is protected on the delagate so the call + * should utilize reflection. + * + */ +public class JSONRootElementProviderEclipseLink + extends AbstractRootElementProvider { + private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(JSONRootElementProviderEclipseLink.class); + + private final JSONRootElementProvider delegate; + + JSONRootElementProviderEclipseLink(Providers ps) throws Exception { + super(ps); + this.delegate = createDelegateInstance(ps); + } + + JSONRootElementProviderEclipseLink(Providers ps, MediaType mt) + throws Exception { + super(ps, mt); + this.delegate = createDelegateInstance(ps, mt); + } + + /** + * Since JSONRootElementProvider constructors are package-private, it cannot + * be instantiated simply, but with reflection. + * + * @param providers + * @return + */ + private static JSONRootElementProvider createDelegateInstance( + Providers providers) throws Exception { + + Constructor constructor = + JSONRootElementProvider.class.getDeclaredConstructor(Providers.class); + constructor.setAccessible(true); + return constructor.newInstance(providers); + } + + /** + * Since JSONRootElementProvider constructors are package-private, it cannot + * be instantiated simply, but with reflection. + * + * @param providers + * @return + */ + private static JSONRootElementProvider createDelegateInstance( + Providers providers, MediaType mediaType) throws Exception { + + Constructor constructor = + JSONRootElementProvider.class.getDeclaredConstructor(Providers.class, + MediaType.class); + constructor.setAccessible(true); + return constructor.newInstance(providers, mediaType); + } + + @Context + @Override + public void setConfiguration(FeaturesAndProperties fp) { + delegate.setConfiguration(fp); + } + + @Override + public boolean isReadable(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return delegate.isReadable(type, genericType, annotations, mediaType); + } + + @Override + public boolean isWriteable(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return delegate.isWriteable(type, genericType, annotations, mediaType); + } + + @Override + protected final Object readFrom(Class type, MediaType mediaType, + Unmarshaller u, InputStream entityStream) throws JAXBException { + try { + return invokeReadFromOnDelegate(type, mediaType, u, entityStream); + } catch (InvocationTargetException e) { + if (e.getCause() instanceof JAXBException) { + throw (JAXBException) e.getCause(); + } else { + throw new RuntimeException(e); + } + } catch (NoSuchMethodException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("JavaReflectionMemberAccess") + private Object invokeReadFromOnDelegate(Class type, + MediaType mediaType, Unmarshaller u, InputStream entityStream) + throws NoSuchMethodException, InvocationTargetException, + IllegalAccessException { + Method method = JSONRootElementProvider.class.getMethod("readFrom", + Class.class, MediaType.class, Unmarshaller.class, InputStream.class); + method.setAccessible(true); + return method.invoke(delegate, type, mediaType, u, entityStream); + } + + @Produces("application/json") + @Consumes("application/json") + public static final class App extends JSONRootElementProviderEclipseLink { + private static final Set SUPPORTED_TYPES = + ImmutableSet.copyOf(ECLIPSELINK_SERIALIZED_TYPES); + + public App(@Context Providers ps) throws Exception { + super(ps, MediaType.APPLICATION_JSON_TYPE); + LOG.debug("Creating {} , supported types: {}", + getClass(), Arrays.toString(SUPPORTED_TYPES.toArray())); + } + + @Override + public boolean isWriteable(Class type, Type genericType, + Annotation[] annotations, MediaType mediaType) { + return isClassSupported(type) + && super.isWriteable(type, genericType, annotations, mediaType); + } + + private boolean isClassSupported(Class type) { + return SUPPORTED_TYPES.contains(type); + } + + @Override + protected void writeTo(Object obj, MediaType mediaType, Charset c, + Marshaller defaultMarshaller, OutputStream entityStream) + throws JAXBException { + LOG.debug("Invoked {}.writeTo() with object type {} media type: {}", + getClass(), obj.getClass(), mediaType); + final Marshaller marshaller = + getMarshallerForMediaType(getMediaType(mediaType), defaultMarshaller); + marshaller.marshal(obj, new OutputStreamWriter(entityStream, c)); + } + + private Marshaller getMarshallerForMediaType(MediaType mediaType, + Marshaller defaultMarshaller) { + // fallback to framework-provided marshaller if mediaType is not found in + // map + return MarshallersHolder.getOrDefault(mediaType, defaultMarshaller); + } + + /** + * Gets the MediaType. If the MediaType is an instance of + * QualitySourceMediaType, e.g. application/json;charset=utf-8, it will + * extract the MediaType for application/json. + * + * @param mediaType The default mediaType + * @return + */ + private MediaType getMediaType(final MediaType mediaType) { + if (mediaType instanceof QualitySourceMediaType) { + QualitySourceMediaType qsMediaType = (QualitySourceMediaType) mediaType; + String type = qsMediaType.getType(); + String subtype = qsMediaType.getSubtype(); + return MediaType.valueOf(type + "/" + subtype); + } + return mediaType; + } + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MapAdapter.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MapAdapter.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MapAdapter.java (date 1513958977000) @@ -0,0 +1,54 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; +import java.util.*; +import java.util.Map.Entry; +import javax.xml.bind.annotation.*; +import javax.xml.bind.annotation.adapters.XmlAdapter; +import org.eclipse.persistence.oxm.annotations.XmlVariableNode; + +public class MapAdapter + extends XmlAdapter> { + + public static class AdaptedMap { + + @XmlVariableNode("key") + List entries = new ArrayList<>(); + + } + + public static class AdaptedEntry { + + @XmlTransient + public String key; + + @XmlValue + public Long value; + + } + + @Override + public AdaptedMap marshal(Map map) { + AdaptedMap adaptedMap = new AdaptedMap(); + + if (map == null) { + return adaptedMap; + } + for (Entry entry : map.entrySet()) { + AdaptedEntry adaptedEntry = new AdaptedEntry(); + adaptedEntry.key = entry.getKey(); + adaptedEntry.value = entry.getValue(); + adaptedMap.entries.add(adaptedEntry); + } + return adaptedMap; + } + + @Override + public Map unmarshal(AdaptedMap adaptedMap) { + List adaptedEntries = adaptedMap.entries; + Map map = new HashMap<>(adaptedEntries.size()); + for (AdaptedEntry adaptedEntry : adaptedEntries) { + map.put(adaptedEntry.key, adaptedEntry.value); + } + return map; + } + +} \ No newline at end of file Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MarshallersHolder.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MarshallersHolder.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/MarshallersHolder.java (date 1513958977000) @@ -0,0 +1,84 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.eclipse.persistence.jaxb.MarshallerProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.MediaType; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import java.util.Arrays; +import java.util.Map; + +import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; +import static javax.ws.rs.core.MediaType.APPLICATION_XML_TYPE; + +public class MarshallersHolder { + private static final Logger LOG = LoggerFactory.getLogger(MarshallersHolder.class); + private static final Map MARSHALLERS = + createMarshallers(); + + private static ImmutableMap createMarshallers() { + return ImmutableMap.of(APPLICATION_JSON_TYPE, + createMarshallerForMediaType(APPLICATION_JSON_TYPE), + APPLICATION_XML_TYPE, + createMarshallerForMediaType(APPLICATION_XML_TYPE)); + } + + private static Marshaller createMarshallerForMediaType(MediaType mediaType) { + try { + // use an arbitrary EclipseLink-serialized class to get EclipseLink's + // JAXBContext + JAXBContext jaxbContext = createJaxbContext( + JAXBContextResolver.ECLIPSELINK_SERIALIZED_TYPES[0]); + + if (mediaType.equals(APPLICATION_JSON_TYPE)) { + return createJsonMarshaller(jaxbContext); + } else if (mediaType.equals(APPLICATION_XML_TYPE)) { + return createXmlMarshaller(jaxbContext); + } else { + throw new IllegalArgumentException("Unknown mediaType: " + mediaType); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static JAXBContext createJaxbContext(Class clazz) throws Exception { + JAXBContextResolver jaxbContextResolver = new JAXBContextResolver(); + return jaxbContextResolver.getContext(clazz); + } + + public static Marshaller createJsonMarshaller(JAXBContext jaxbContext) + throws JAXBException { + Marshaller marshaller = jaxbContext.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + marshaller.setProperty(MarshallerProperties.MEDIA_TYPE, "application/json"); + marshaller.setProperty(MarshallerProperties.JSON_INCLUDE_ROOT, true); + marshaller.setProperty(MarshallerProperties.JSON_MARSHAL_EMPTY_COLLECTIONS, + false); + + LOG.debug("Created JSON marshaller (EclipseLink) {}", marshaller); + return marshaller; + } + + public static Marshaller createXmlMarshaller(JAXBContext jaxbContext) + throws JAXBException { + Marshaller marshaller = jaxbContext.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + + return marshaller; + } + + public static Marshaller getOrDefault(MediaType mediaType, + Marshaller defaultMarshaller) { + LOG.debug( + "Invoking {}.getOrDefault() with media type: {}, available marshallers: {}", + MarshallersHolder.class, mediaType, + Arrays.toString(MARSHALLERS.entrySet().toArray())); + return MARSHALLERS.getOrDefault(mediaType, defaultMarshaller); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/ServiceFinder.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/ServiceFinder.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/ServiceFinder.java (date 1513958977000) @@ -0,0 +1,647 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper; + +import com.sun.jersey.core.osgi.OsgiRegistry; +import com.sun.jersey.core.reflection.ReflectionHelper; +import com.sun.jersey.impl.SpiMessages; +import com.sun.jersey.spi.service.ServiceConfigurationError; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.ext.MessageBodyWriter; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.net.URLConnection; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.util.*; + +/** + * This class is almost a copy of Jersey's + * {@link com.sun.jersey.spi.service.ServiceFinder} with a few exceptions like + * logging methods and public methods that was not required to copy, with the + * intention to replace its IteratorProvider with a custom implementation, + * + * see + * {@link com.sun.jersey.spi.service.ServiceFinder#setIteratorProvider(com.sun.jersey.spi.service.ServiceFinder.ServiceIteratorProvider)} + *
+ *
+ * Given that the custom implementation should be as close as possible to the + * original + * {@link com.sun.jersey.spi.service.ServiceFinder.DefaultServiceIteratorProvider} + * and both the iterators + * ({@link com.sun.jersey.spi.service.ServiceFinder.LazyObjectIterator} and + * {@link com.sun.jersey.spi.service.ServiceFinder.LazyClassIterator} and its + * accompanying code that the iterators were based on is private, a copy was + * needed, unfortunately.
+ *
+ * The reason why the custom implementation is needed: Despite Jersey's + * documentation clearly declares that a user created {@link MessageBodyWriter} + * like the {@link JSONRootElementProviderEclipseLink} would be chosen in favor + * of the Providers defined by Jersey while serializing response objects, on the + * contrary, it does not work like that.
+ * Apparently, the iteration order defined by the return value of the method + * {@link DefaultServiceIteratorProvider#createClassIterator(Class, String, ClassLoader, boolean)} + * matters in terms of choosing the preferred {@link MessageBodyWriter}.
+ *
+ * See the method {@link AbstractLazyIterator#reorderURLsIfNeeded(List)} for + * details. + * + * + */ +public final class ServiceFinder { + private static final Logger LOG = + LoggerFactory.getLogger(ServiceFinder.class); + private static final String MODULE_VERSION = "META-INF/jersey-module-version"; + + private static final String PREFIX = "META-INF/services/"; + + private static final String MODULE_VERSION_VALUE = getModuleVersion(); + + private static final Set MODULES_BLACKLIST; + + static { + MODULES_BLACKLIST = new HashSet() { + { + + // Jersey + add("jersey-client"); + add("jersey-core"); + add("jersey-gf-server"); + add("jersey-gf-servlet"); + add("jersey-gf-statsproviders"); + add("jersey-grizzly"); + add("jersey-json"); + add("jersey-moxy"); + add("jersey-multipart"); + add("jersey-server"); + add("jersey-servlet"); + add("jersey-statsproviders"); + + // GF + add("glassfish-embedded"); + + } + }; + + final OsgiRegistry osgiRegistry = + ReflectionHelper.getOsgiRegistryInstance(); + + if (osgiRegistry != null) { + LOG.debug("Running in an OSGi environment"); + } else { + LOG.debug("Running in a non-OSGi environment"); + } + } + + private static String getModuleVersion() { + try { + String resource = + ServiceFinder.class.getName().replace(".", "/") + ".class"; + URL url = getResource(ServiceFinder.class.getClassLoader(), resource); + if (url == null) { + LOG.debug("Error getting {} class as a resource", + ServiceFinder.class.getName()); + return null; + } + + return getJerseyModuleVersion(ServiceFinder.class.getName(), url); + } catch (IOException ioe) { + LOG.debug("Error loading META-INF/jersey-module-version associated with " + + ServiceFinder.class.getName(), ioe); + return null; + } + } + + private static List filterServiceURLsWithVersion(String serviceName, + Enumeration serviceUrls) { + if (MODULE_VERSION_VALUE == null || !serviceUrls.hasMoreElements()) { + return Collections.list(serviceUrls); + } + + final List urls = Collections.list(serviceUrls); + final ListIterator li = urls.listIterator(); + while (li.hasNext()) { + final URL url = li.next(); + + if (isServiceInBlacklistedModule(url)) { + String jerseyModuleVersion = getJerseyModuleVersion(serviceName, url); + + if (jerseyModuleVersion != null) { + if (!MODULE_VERSION_VALUE.equals(jerseyModuleVersion)) { + li.remove(); + } + } + } + } + return urls; + } + + private static boolean isServiceInBlacklistedModule(final URL serviceUrl) { + final String service = serviceUrl.toString(); + for (final String module : MODULES_BLACKLIST) { + if (service.contains(module)) { + return true; + } + } + return false; + } + + private static String getJerseyModuleVersion(final String serviceName, + final URL serviceUrl) { + InputStream stream = null; + + try { + final String url = serviceUrl.toString(); + final String resource = + url.endsWith("class") ? serviceName.replace(".", "/") + ".class" + : serviceName; + final URL moduleVersionURL = + new URL(url.replace(resource, MODULE_VERSION)); + + stream = moduleVersionURL.openStream(); + return new BufferedReader(new InputStreamReader(stream)).readLine(); + } catch (IOException ioe) { + LOG.debug( + "Error loading META-INF/jersey-module-version associated with {}", + ServiceFinder.class.getName()); + return null; + } finally { + if (stream != null) { + try { + stream.close(); + } catch (IOException ioe) { + LOG.debug("Error closing stream associated with {}", + ServiceFinder.class.getName()); + } + } + } + } + + private static URL getResource(ClassLoader loader, String name) + throws IOException { + if (loader == null) + return getResource(name); + else { + final URL resource = loader.getResource(name); + if (resource != null) { + return resource; + } else { + return getResource(name); + } + } + } + + private static URL getResource(String name) throws IOException { + if (ServiceFinder.class.getClassLoader() != null) + return ServiceFinder.class.getClassLoader().getResource(name); + else + return ClassLoader.getSystemResource(name); + } + + private static Enumeration getResources(ClassLoader loader, String name) + throws IOException { + if (loader == null) { + return getResources(name); + } else { + final Enumeration resources = loader.getResources(name); + if ((resources != null) && resources.hasMoreElements()) { + return resources; + } else { + return getResources(name); + } + } + } + + private static Enumeration getResources(String name) throws IOException { + if (ServiceFinder.class.getClassLoader() != null) + return ServiceFinder.class.getClassLoader().getResources(name); + else + return ClassLoader.getSystemResources(name); + } + + private static void fail(String serviceName, String msg, Throwable cause) + throws ServiceConfigurationError { + ServiceConfigurationError sce = + new ServiceConfigurationError(serviceName + ": " + msg); + sce.initCause(cause); + throw sce; + } + + private static void fail(String serviceName, String msg) + throws ServiceConfigurationError { + throw new ServiceConfigurationError(serviceName + ": " + msg); + } + + private static void fail(String serviceName, URL u, int line, String msg) + throws ServiceConfigurationError { + fail(serviceName, u + ":" + line + ": " + msg); + } + + /** + * Parse a single line from the given configuration file, adding the name on + * the line to both the names list and the returned set iff the name is not + * already a member of the returned set. + */ + private static int parseLine(String serviceName, URL u, BufferedReader r, + int lc, List names, Set returned) + throws IOException, ServiceConfigurationError { + String ln = r.readLine(); + if (ln == null) { + return -1; + } + int ci = ln.indexOf('#'); + if (ci >= 0) + ln = ln.substring(0, ci); + ln = ln.trim(); + int n = ln.length(); + if (n != 0) { + if ((ln.indexOf(' ') >= 0) || (ln.indexOf('\t') >= 0)) + fail(serviceName, u, lc, SpiMessages.ILLEGAL_CONFIG_SYNTAX()); + int cp = ln.codePointAt(0); + if (!Character.isJavaIdentifierStart(cp)) + fail(serviceName, u, lc, SpiMessages.ILLEGAL_PROVIDER_CLASS_NAME(ln)); + for (int i = Character.charCount(cp); i < n; i += + Character.charCount(cp)) { + cp = ln.codePointAt(i); + if (!Character.isJavaIdentifierPart(cp) && (cp != '.')) + fail(serviceName, u, lc, SpiMessages.ILLEGAL_PROVIDER_CLASS_NAME(ln)); + } + if (!returned.contains(ln)) { + names.add(ln); + returned.add(ln); + } + } + return lc + 1; + } + + /** + * Parse the content of the given URL as a provider-configuration file. + * + * @param serviceName The service class for which providers are being sought; + * used to construct error detail strings + * @param u The URL naming the configuration file to be parsed + * @param returned A Set containing the names of provider classes that have + * already been returned. This set will be updated to contain the + * names that will be yielded from the returned Iterator. + * @return A (possibly empty) Iterator that will yield the + * provider-class names in the given configuration file that are not + * yet members of the returned set + * @throws ServiceConfigurationError If an I/O error occurs while reading from + * the given URL, or if a configuration-file format error is + * detected + */ + @SuppressWarnings({ "StatementWithEmptyBody" }) + private static Iterator parse(String serviceName, URL u, + Set returned) throws ServiceConfigurationError { + InputStream in = null; + BufferedReader r = null; + ArrayList names = new ArrayList(); + try { + URLConnection uConn = u.openConnection(); + uConn.setUseCaches(false); + in = uConn.getInputStream(); + r = new BufferedReader(new InputStreamReader(in, "utf-8")); + int lc = 1; + while ((lc = parseLine(serviceName, u, r, lc, names, returned)) >= 0) + ; + } catch (IOException x) { + fail(serviceName, ": " + x); + } finally { + try { + if (r != null) + r.close(); + if (in != null) + in.close(); + } catch (IOException y) { + fail(serviceName, ": " + y); + } + } + return names.iterator(); + } + + private static class AbstractLazyIterator { + final Class service; + final String serviceName; + final ClassLoader loader; + final boolean ignoreOnClassNotFound; + + Enumeration configs = null; + Iterator pending = null; + Set returned = new TreeSet(); + String nextName = null; + + private AbstractLazyIterator(Class service, String serviceName, + ClassLoader loader, boolean ignoreOnClassNotFound) { + this.service = service; + this.serviceName = serviceName; + this.loader = loader; + this.ignoreOnClassNotFound = ignoreOnClassNotFound; + + final String fullName = PREFIX + serviceName; + + try { + List urls = filterServiceURLsWithVersion(fullName, + getResources(loader, fullName)); + logConfig(urls); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + private void logConfig(List urls) { + StringBuilder sb = new StringBuilder(); + for (URL url : urls) { + sb.append(url.toString()).append(", "); + } + + LOG.debug("ServiceFinder config for class: {}, configuration URLs: " + + serviceName, sb.toString()); + } + + protected final void setConfigs() { + if (configs == null) { + try { + final String fullName = PREFIX + serviceName; + List urls = filterServiceURLsWithVersion(fullName, + getResources(loader, fullName)); + reorderURLsIfNeeded(urls); + configs = Collections.enumeration(urls); + } catch (IOException x) { + fail(serviceName, ": " + x); + } + } + } + + /** + * Reorder the provided urls list so that the jar file + * hadoop-yarn-server-resourcemanager.jar is always in the first place in + * the list only if the service is {@link MessageBodyWriter} + * + * @param urls + */ + private void reorderURLsIfNeeded(List urls) { + // format of jar file name: + // hadoop-yarn-server-resourcemanager-3.1.0-SNAPSHOT.jar + if (service.equals(MessageBodyWriter.class)) { + int indexOfResource = + getIndexOfResource(urls, "hadoop-yarn-server-resourcemanager"); + if (indexOfResource != -1) { + Collections.swap(urls, 0, indexOfResource); + } + } + } + + private int getIndexOfResource(List urls, String resourceName) { + for (int i = 0; i < urls.size(); i++) { + URL url = urls.get(i); + if (url != null && url.getPath() != null + && url.getPath().matches(".*" + resourceName + ".*\\.jar.*")) { + return i; + } + } + return -1; + } + + public boolean hasNext() throws ServiceConfigurationError { + if (nextName != null) { + return true; + } + setConfigs(); + + while (nextName == null) { + while ((pending == null) || !pending.hasNext()) { + if (!configs.hasMoreElements()) { + return false; + } + pending = parse(serviceName, configs.nextElement(), returned); + } + nextName = pending.next(); + if (ignoreOnClassNotFound) { + try { + AccessController.doPrivileged(ReflectionHelper + .classForNameWithExceptionPEA(nextName, loader)); + } catch (ClassNotFoundException ex) { + handleClassNotFoundException(); + } catch (PrivilegedActionException pae) { + final Throwable thrown = pae.getException(); + if (thrown instanceof ClassNotFoundException) { + handleClassNotFoundException(); + } else if (thrown instanceof NoClassDefFoundError) { + // Dependent class of provider not found + // This assumes that ex.getLocalizedMessage() returns + // the name of a dependent class that is not found + LOG.debug(SpiMessages.DEPENDENT_CLASS_OF_PROVIDER_NOT_FOUND( + thrown.getLocalizedMessage(), nextName, service)); + nextName = null; + } else if (thrown instanceof ClassFormatError) { + // Dependent class of provider not found + LOG.debug(SpiMessages.DEPENDENT_CLASS_OF_PROVIDER_FORMAT_ERROR( + thrown.getLocalizedMessage(), nextName, service)); + nextName = null; + } else if (thrown instanceof RuntimeException) { + throw (RuntimeException) thrown; + } else { + throw new IllegalStateException(thrown); + } + } + } + } + return true; + } + + private void handleClassNotFoundException() { + // Provider implementation not found + LOG.debug(SpiMessages.PROVIDER_NOT_FOUND(nextName, service)); + nextName = null; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } + + private static final class LazyClassIterator extends + ServiceFinder.AbstractLazyIterator implements Iterator> { + + private LazyClassIterator(Class service, String serviceName, + ClassLoader loader, boolean ignoreOnClassNotFound) { + super(service, serviceName, loader, ignoreOnClassNotFound); + } + + @SuppressWarnings("unchecked") + public Class next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + String cn = nextName; + nextName = null; + try { + + Class tClass = AccessController.doPrivileged( + ReflectionHelper. classForNameWithExceptionPEA(cn, loader)); + LOG.debug("Loading next class: {}", tClass.getName()); + + return tClass; + + } catch (ClassNotFoundException ex) { + fail(serviceName, SpiMessages.PROVIDER_NOT_FOUND(cn, service)); + } catch (PrivilegedActionException pae) { + + final Throwable thrown = pae.getCause(); + + if (thrown instanceof ClassNotFoundException) { + fail(serviceName, SpiMessages.PROVIDER_NOT_FOUND(cn, service)); + } else if (thrown instanceof NoClassDefFoundError) { + fail(serviceName, SpiMessages.DEPENDENT_CLASS_OF_PROVIDER_NOT_FOUND( + thrown.getLocalizedMessage(), cn, service)); + } else if (thrown instanceof ClassFormatError) { + fail(serviceName, + SpiMessages.DEPENDENT_CLASS_OF_PROVIDER_FORMAT_ERROR( + thrown.getLocalizedMessage(), cn, service)); + } else { + fail(serviceName, SpiMessages.PROVIDER_CLASS_COULD_NOT_BE_LOADED(cn, + service, thrown.getLocalizedMessage()), thrown); + } + } + + return null; /* This cannot happen */ + } + } + + private static final class LazyObjectIterator + extends ServiceFinder.AbstractLazyIterator implements Iterator { + + private T t; + + private LazyObjectIterator(Class service, String serviceName, + ClassLoader loader, boolean ignoreOnClassNotFound) { + super(service, serviceName, loader, ignoreOnClassNotFound); + } + + @Override + public boolean hasNext() throws ServiceConfigurationError { + if (nextName != null) { + return true; + } + setConfigs(); + + while (nextName == null) { + while ((pending == null) || !pending.hasNext()) { + if (!configs.hasMoreElements()) { + return false; + } + pending = parse(serviceName, configs.nextElement(), returned); + } + nextName = pending.next(); + try { + t = service.cast(AccessController + .doPrivileged(ReflectionHelper + .classForNameWithExceptionPEA(nextName, loader)) + .newInstance()); + + } catch (InstantiationException ex) { + if (ignoreOnClassNotFound) { + LOG.debug(SpiMessages.PROVIDER_COULD_NOT_BE_CREATED(nextName, + service, ex.getLocalizedMessage())); + nextName = null; + } else { + fail(serviceName, SpiMessages.PROVIDER_COULD_NOT_BE_CREATED( + nextName, service, ex.getLocalizedMessage()), ex); + } + } catch (IllegalAccessException ex) { + fail(serviceName, SpiMessages.PROVIDER_COULD_NOT_BE_CREATED(nextName, + service, ex.getLocalizedMessage()), ex); + + } catch (ClassNotFoundException ex) { + handleClassNotFoundException(); + } catch (NoClassDefFoundError ex) { + // Dependent class of provider not found + if (ignoreOnClassNotFound) { + // This assumes that ex.getLocalizedMessage() returns + // the name of a dependent class that is not found + LOG.debug(SpiMessages.DEPENDENT_CLASS_OF_PROVIDER_NOT_FOUND( + ex.getLocalizedMessage(), nextName, service)); + nextName = null; + } else { + fail(serviceName, SpiMessages.DEPENDENT_CLASS_OF_PROVIDER_NOT_FOUND( + ex.getLocalizedMessage(), nextName, service), ex); + } + + } catch (PrivilegedActionException pae) { + final Throwable cause = pae.getCause(); + if (cause instanceof ClassNotFoundException) { + handleClassNotFoundException(); + } else if (cause instanceof ClassFormatError) { + // Dependent class of provider not found + if (ignoreOnClassNotFound) { + LOG.debug(SpiMessages.DEPENDENT_CLASS_OF_PROVIDER_FORMAT_ERROR( + cause.getLocalizedMessage(), nextName, service)); + nextName = null; + } else { + fail(serviceName, + SpiMessages.DEPENDENT_CLASS_OF_PROVIDER_FORMAT_ERROR( + cause.getLocalizedMessage(), nextName, service), + cause); + } + } else { + fail(serviceName, SpiMessages.PROVIDER_COULD_NOT_BE_CREATED( + nextName, service, cause.getLocalizedMessage()), cause); + } + } + } + return true; + } + + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + String cn = nextName; + nextName = null; + return t; + } + + private void handleClassNotFoundException() + throws ServiceConfigurationError { + if (ignoreOnClassNotFound) { + // Provider implementation not found + LOG.debug(SpiMessages.PROVIDER_NOT_FOUND(nextName, service)); + nextName = null; + } else { + fail(serviceName, SpiMessages.PROVIDER_NOT_FOUND(nextName, service)); + } + } + } + + /** + * The default service iterator provider that looks up provider classes in + * META-INF/services files. + *

+ * This class may utilized if a + * {@link com.sun.jersey.spi.service.ServiceFinder.ServiceIteratorProvider} + * needs to reuse the default implementation. + * + * @param the type of service. + */ + public static final class DefaultServiceIteratorProvider extends + com.sun.jersey.spi.service.ServiceFinder.ServiceIteratorProvider { + + @Override + public Iterator createIterator(Class service, String serviceName, + ClassLoader loader, boolean ignoreOnClassNotFound) { + return new ServiceFinder.LazyObjectIterator(service, serviceName, loader, + ignoreOnClassNotFound); + } + + @Override + public Iterator> createClassIterator(Class service, + String serviceName, ClassLoader loader, boolean ignoreOnClassNotFound) { + return new ServiceFinder.LazyClassIterator(service, serviceName, loader, + ignoreOnClassNotFound); + } + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/META-INF/services/javax.ws.rs.ext.MessageBodyWriter IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/META-INF/services/javax.ws.rs.ext.MessageBodyWriter (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/META-INF/services/javax.ws.rs.ext.MessageBodyWriter (date 1513958977000) @@ -0,0 +1,1 @@ +org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.JSONRootElementProviderEclipseLink$App \ No newline at end of file Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppInfoVerifications.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppInfoVerifications.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppInfoVerifications.java (date 1513958977000) @@ -0,0 +1,80 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; + +import static org.junit.Assert.*; + +public class AppInfoVerifications { + public static void verify(ElementWrapper info, RMApp app) { + WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() + .toString(), info.getString("id")); + WebServicesTestUtils.checkStringMatch("user", app.getUser(), info.getString("user")); + WebServicesTestUtils.checkStringMatch("name", app.getName(), info.getString("name")); + WebServicesTestUtils.checkStringMatch("applicationType", + app.getApplicationType(), info.getString("applicationType")); + WebServicesTestUtils.checkStringMatch("queue", app.getQueue(), info.getString("queue")); + assertEquals("priority doesn't match", 0, (int) info.getInt("priority")); + WebServicesTestUtils.checkStringMatch("state", app.getState().toString(), + info.getString("state")); + WebServicesTestUtils.checkStringMatch("finalStatus", app + .getFinalApplicationStatus().toString(), info.getString("finalStatus")); + assertEquals("progress doesn't match", 0, info.getFloat("progress"), 0.0); + if ("UNASSIGNED".equals(info.getString("trackingUI"))) { + WebServicesTestUtils.checkStringMatch("trackingUI", "UNASSIGNED", + info.getString("trackingUI")); + } + WebServicesTestUtils.checkStringEqual("diagnostics", + app.getDiagnostics().toString(), info.getString("diagnostics")); + assertEquals("clusterId doesn't match", + ResourceManager.getClusterTimeStamp(), (long)info.getLong("clusterId")); + assertEquals("startedTime doesn't match", app.getStartTime(), (long)info.getLong("startedTime")); + assertEquals("finishedTime doesn't match", app.getFinishTime(), + (long)info.getLong("finishedTime")); + assertTrue("elapsed time not greater than 0", info.getLong("elapsedTime") > 0); + WebServicesTestUtils.checkStringMatch("amHostHttpAddress", app + .getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(), + info.getString("amHostHttpAddress")); + assertTrue("amContainerLogs doesn't match", + info.getString("amContainerLogs").startsWith("http://")); + assertTrue("amContainerLogs doesn't contain user info", + info.getString("amContainerLogs").endsWith("/" + app.getUser())); + assertEquals("allocatedMB doesn't match", 1024, (int)info.getInt("allocatedMB")); + assertEquals("allocatedVCores doesn't match", 1, (int)info.getInt("allocatedVCores")); + assertEquals("queueUsagePerc doesn't match", 50.0f, info.getFloat("queueUsagePercentage"), 0.01f); + assertEquals("clusterUsagePerc doesn't match", 50.0f, info.getFloat("clusterUsagePercentage"), 0.01f); + assertEquals("numContainers doesn't match", 1, (int) info.getInt("runningContainers")); + assertNotNull("preemptedResourceSecondsMap should not be null", info.getChild + ("preemptedResourceSecondsMap")); + assertEquals("preemptedResourceMB doesn't match", app + .getRMAppMetrics().getResourcePreempted().getMemorySize(), + (int)info.getInt("preemptedResourceMB")); + assertEquals("preemptedResourceVCores doesn't match", app + .getRMAppMetrics().getResourcePreempted().getVirtualCores(), + (int) info.getInt("preemptedResourceVCores")); + assertEquals("numNonAMContainerPreempted doesn't match", app + .getRMAppMetrics().getNumNonAMContainersPreempted(), + (int)info.getInt("numNonAMContainerPreempted")); + assertEquals("numAMContainerPreempted doesn't match", app + .getRMAppMetrics().getNumAMContainersPreempted(), + (int) info.getInt("numAMContainerPreempted")); + assertEquals("Log aggregation Status doesn't match", app + .getLogAggregationStatusForAppReport().toString(), + info.getString("logAggregationStatus")); + assertEquals("unmanagedApplication doesn't match", app + .getApplicationSubmissionContext().getUnmanagedAM(), + info.getBoolean("unmanagedApplication")); + assertEquals("unmanagedApplication doesn't match", + app.getApplicationSubmissionContext().getNodeLabelExpression(), + info.getStringSafely("appNodeLabelExpression")); + assertEquals("unmanagedApplication doesn't match", + app.getAMResourceRequests().get(0).getNodeLabelExpression(), + info.getStringSafely("amNodeLabelExpression")); + assertEquals("amRPCAddress", + AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()), + info.getStringSafely("amRPCAddress")); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ResourceRequestsVerifications.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ResourceRequestsVerifications.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ResourceRequestsVerifications.java (date 1513958977000) @@ -0,0 +1,145 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ResourceRequestsVerifications { + private final ResourceRequest resourceRequest; + private final ElementWrapper requestInfo; + private final Map customResourceTypes; + private final List expectedCustomResourceTypes; + + ResourceRequestsVerifications(Builder builder) { + this.resourceRequest = builder.resourceRequest; + this.requestInfo = builder.requestInfo; + this.customResourceTypes = builder.customResourceTypes; + this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes; + } + + public static void verify(ElementWrapper requestInfo, ResourceRequest rr) { + createDefaultBuilder(requestInfo, rr) + .build() + .verify(); + } + + public static void verifyWithCustomResourceTypes(ElementWrapper requestInfo, + ResourceRequest resourceRequest, List expectedResourceTypes) { + + createDefaultBuilder(requestInfo, resourceRequest) + .withExpectedCustomResourceTypes(expectedResourceTypes) + .withCustomResourceTypes(extractActualCustomResourceTypes(requestInfo, + expectedResourceTypes)) + .build() + .verify(); + } + + private static Builder createDefaultBuilder(ElementWrapper requestInfo, + ResourceRequest resourceRequest) { + return new ResourceRequestsVerifications.Builder() + .withRequest(resourceRequest) + .withRequestInfo(requestInfo); + } + + private static Map extractActualCustomResourceTypes( + ElementWrapper requestInfo, List expectedResourceTypes) { + ElementWrapper capability = requestInfo.getChild("capability"); + + return expectedResourceTypes.stream() + .collect(HashMap::new, + (map, value) -> map.put(value, + extractCustomResorceTypeValue(capability, value)), + HashMap::putAll); + } + + private static Long extractCustomResorceTypeValue(ElementWrapper capability, + String resourceType) { + if (capability.hasChild(resourceType)) { + return capability.getLong(resourceType); + } + return null; + } + + private void verify() { + assertEquals("nodeLabelExpression doesn't match", + resourceRequest.getNodeLabelExpression(), requestInfo.getString("nodeLabelExpression")); + assertEquals("numContainers doesn't match", resourceRequest.getNumContainers(), + (int)requestInfo.getInt("numContainers")); + assertEquals("relaxLocality doesn't match", resourceRequest.getRelaxLocality(), + requestInfo.getBoolean("relaxLocality")); + assertEquals("priority does not match", resourceRequest.getPriority().getPriority(), + (int)requestInfo.getInt("priority")); + assertEquals("resourceName does not match", resourceRequest.getResourceName(), + requestInfo.getString("resourceName")); + assertEquals("memory does not match", + resourceRequest.getCapability().getMemorySize(), (long)requestInfo.getChild("capability").getLong("memory")); + assertEquals("vCores does not match", + resourceRequest.getCapability().getVirtualCores(), (long)requestInfo.getChild("capability").getLong("vCores")); + + for (String expectedCustomResourceType : expectedCustomResourceTypes) { + assertTrue( + "Custom resource type " + expectedCustomResourceType + + " cannot be found!", + customResourceTypes.containsKey(expectedCustomResourceType)); + + Long resourceValue = customResourceTypes.get(expectedCustomResourceType); + assertNotNull("Resource value should not be null!", resourceValue); + } + + ElementWrapper executionTypeRequest = + requestInfo.getChild("executionTypeRequest"); + assertEquals("executionType does not match", + resourceRequest.getExecutionTypeRequest().getExecutionType().name(), + executionTypeRequest.getString("executionType")); + assertEquals("enforceExecutionType does not match", + resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(), + executionTypeRequest.getBoolean("enforceExecutionType")); + } + + public static final class Builder { + private List expectedCustomResourceTypes; + private Map customResourceTypes; + private ResourceRequest resourceRequest; + private ElementWrapper requestInfo; + + Builder() { + } + + public static Builder create() { + return new Builder(); + } + + public Builder withExpectedCustomResourceTypes( + List expectedCustomResourceTypes) { + this.expectedCustomResourceTypes = expectedCustomResourceTypes; + return this; + } + + public Builder withCustomResourceTypes( + Map customResourceTypes) { + this.customResourceTypes = customResourceTypes; + return this; + } + + public Builder withRequest(ResourceRequest resourceRequest) { + this.resourceRequest = resourceRequest; + return this; + } + + public Builder withRequestInfo(ElementWrapper requestInfo) { + this.requestInfo = requestInfo; + return this; + } + + public ResourceRequestsVerifications build() { + return new ResourceRequestsVerifications(this); + } + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestMarshalAppInfoWithEclipseLinkJaxbProvider.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestMarshalAppInfoWithEclipseLinkJaxbProvider.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestMarshalAppInfoWithEclipseLinkJaxbProvider.java (date 1513958977000) @@ -0,0 +1,283 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; +import org.apache.hadoop.http.JettyUtils; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.*; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ErrorCollector; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import javax.ws.rs.core.MediaType; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.function.Consumer; + +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver.JSON_CONFIG_DEFAULT; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.MarshallersHolder.createJsonMarshaller; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.MarshallersHolder.createXmlMarshaller; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) +public class TestMarshalAppInfoWithEclipseLinkJaxbProvider { + + @Rule + public ErrorCollector collector = new ErrorCollector(); + + @Parameters + public static Collection marshallers() throws Exception { + JAXBContext eclipseLinkJaxbContext = createEclipseLinkJaxbContext(); + Marshaller xmlMarshaller = createXmlMarshaller(eclipseLinkJaxbContext); + Marshaller jsonMarshaller = createJsonMarshaller(eclipseLinkJaxbContext); + + JAXBContext normalJaxbContext = createNormalJaxbContext(); + Marshaller normalXmlMarshaller = + createNormalXmlMarshaller(normalJaxbContext); + Marshaller normalJsonMarshaller = + createNormalJsonMarshaller(normalJaxbContext); + + return Arrays.asList(new Object[][] { { xmlMarshaller, jsonMarshaller }, + { normalXmlMarshaller, normalJsonMarshaller } }); + } + + private Marshaller xmlMarshaller; + private Object jsonMarshaller; + + public TestMarshalAppInfoWithEclipseLinkJaxbProvider(Marshaller xmlMarshaller, + Object jsonMarshaller) { + this.xmlMarshaller = xmlMarshaller; + this.jsonMarshaller = jsonMarshaller; + } + + private static JAXBContext createEclipseLinkJaxbContext() throws Exception { + JAXBContextResolver jaxbContextResolver = new JAXBContextResolver(); + return jaxbContextResolver.getContext(AppInfo.class); + } + + private static JAXBContext createNormalJaxbContext() throws JAXBException { + return new JSONJAXBContext(JSON_CONFIG_DEFAULT, AppInfo.class); + } + + private static Marshaller createNormalXmlMarshaller(JAXBContext jaxbContext) + throws JAXBException { + return jaxbContext.createMarshaller(); + } + + private static Marshaller createNormalJsonMarshaller(JAXBContext jaxbContext) throws JAXBException { + return jaxbContext.createMarshaller(); + } + + private AppInfo createAppInfoWithEmptyCollections() { + AppInfo appInfo = new AppInfo(); + appInfo.setPreemptedResourceSecondsMap(Maps.newHashMap()); + appInfo.setResourceSecondsMap(Maps.newHashMap()); + appInfo.setResourceRequests(Lists.newArrayList()); + return appInfo; + } + + private AppInfo createAppInfoWithNotEmptyCollections() { + AppInfo appInfo = new AppInfo(); + appInfo.setPreemptedResourceSecondsMap(ImmutableMap.of("bla", 2L)); + appInfo.setResourceSecondsMap(ImmutableMap.of("bla", 12L)); + appInfo.setResourceRequests(ImmutableList.of(new ResourceRequestInfo())); + return appInfo; + } + + private void assertHasChild(ElementWrapper appInfoWrapper, String child) { + String message = + "AppInfo " + appInfoWrapper + " does not have child: " + child; + collector.checkThat(message, appInfoWrapper.hasChild(child), equalTo(true)); + } + + private void assertDoesNotHaveChild(ElementWrapper appInfoWrapper, + String child) { + String message = + "AppInfo " + appInfoWrapper + " should not have child: " + child; + collector.checkThat(message, appInfoWrapper.hasChild(child), + equalTo(false)); + } + + private void assertChildHasValues(ElementWrapper appInfoWrapper, String child, + Map values) { + collector.checkThat( + "AppInfo " + appInfoWrapper + " does not have child: " + child, + appInfoWrapper.hasChild(child), equalTo(true)); + + ElementWrapper mapKeys = appInfoWrapper.getChild(child); + collector + .checkThat( + String.format("app.%s: %s does not have expected size: %d", child, + mapKeys, values.size()), + mapKeys.length(), equalTo(values.size())); + + for (Map.Entry expectedEntry : values.entrySet()) { + String nameOfMapKey = expectedEntry.getKey(); + Long expectedMapValue = expectedEntry.getValue(); + collector.checkThat(mapKeys.getLong(nameOfMapKey), + equalTo(expectedMapValue)); + } + + } + + private WebResource createMockResource() { + WebResource mockResource = mock(WebResource.class); + when(mockResource.toString()).thenReturn("dummyPath"); + return mockResource; + } + + private BufferedClientResponse createMockResponse(StringWriter sw, + MediaType mediaType) throws JSONException { + BufferedClientResponse mockResponse = mock(BufferedClientResponse.class); + + MediaType mockMediaType = mock(MediaType.class); + when(mockMediaType.toString()) + .thenReturn(mediaType + "; " + JettyUtils.UTF_8); + + when(mockResponse.getType()).thenReturn(mockMediaType); + when(mockResponse.getEntity(eq(String.class))).thenReturn(sw.toString()); + + if (mediaType.equals(MediaType.APPLICATION_JSON_TYPE)) { + JSONObject jsonObject = new JSONObject(sw.toString()); + when(mockResponse.getEntity(eq(JSONObject.class))).thenReturn(jsonObject); + } + + return mockResponse; + } + + private Consumer emptyCollectionsVerifier() { + return responseAdapter -> { + ElementWrapper appInfoWrapper = responseAdapter.getElement("app"); + + assertHasChild(appInfoWrapper, "preemptedResourceSecondsMap"); + assertHasChild(appInfoWrapper, "resourceSecondsMap"); + assertDoesNotHaveChild(appInfoWrapper, "resourceRequests"); + }; + } + + private Consumer notEmptyCollectionsVerifier() { + return responseAdapter -> { + ElementWrapper appInfoWrapper = responseAdapter.getElement("app"); + + assertChildHasValues(appInfoWrapper, "preemptedResourceSecondsMap", + ImmutableMap.of("bla", 2L)); + assertChildHasValues(appInfoWrapper, "resourceSecondsMap", + ImmutableMap.of("bla", 12L)); + assertHasChild(appInfoWrapper, "resourceRequests"); + }; + } + + private void marshalJson(AppInfo appInfo, StringWriter sw) + throws JAXBException { + if (jsonMarshaller instanceof JSONMarshaller) { + ((JSONMarshaller) jsonMarshaller).marshallToJSON(appInfo, sw); + } else if (jsonMarshaller instanceof Marshaller) { + ((Marshaller) jsonMarshaller).marshal(appInfo, sw); + } else { + throw new IllegalStateException( + "Wrong type of JSON marshaller: " + jsonMarshaller.getClass()); + } + } + + @Test + public void testMarshalEmptyCollectionsXmlEclipseLinkJaxbContext() + throws Exception { + AppInfo appInfo = createAppInfoWithEmptyCollections(); + StringWriter sw = new StringWriter(); + xmlMarshaller.marshal(appInfo, sw); + + WebResource mockResource = createMockResource(); + BufferedClientResponse mockResponse = + createMockResponse(sw, MediaType.APPLICATION_XML_TYPE); + + XmlCustomResourceTypeTestCase testCase = + new XmlCustomResourceTypeTestCase(mockResource, mockResponse); + testCase.verify(emptyCollectionsVerifier()); + } + + /** + * Note: does not make sense to check normal jersey Marshaller with not empty + * collections on {@link AppInfo} since + * @XmlJavaTypeAdapter and @XmlPath are altering the marshalling behavior that + * the Jersey Marshaller could not handle! + * @throws Exception + */ + @Test + public void testMarshalNotEmptyCollectionsXml() throws Exception { + JAXBContext eclipseLinkJaxbContext = createEclipseLinkJaxbContext(); + Marshaller xmlMarshaller = createXmlMarshaller(eclipseLinkJaxbContext); + + AppInfo appInfo = createAppInfoWithNotEmptyCollections(); + StringWriter sw = new StringWriter(); + xmlMarshaller.marshal(appInfo, sw); + + WebResource mockResource = createMockResource(); + BufferedClientResponse mockResponse = + createMockResponse(sw, MediaType.APPLICATION_XML_TYPE); + + XmlCustomResourceTypeTestCase testCase = + new XmlCustomResourceTypeTestCase(mockResource, mockResponse); + testCase.verify(notEmptyCollectionsVerifier()); + } + + + @Test + public void testMarshalEmptyCollectionsJson() throws Exception { + AppInfo appInfo = createAppInfoWithEmptyCollections(); + StringWriter sw = new StringWriter(); + marshalJson(appInfo, sw); + + WebResource mockResource = createMockResource(); + BufferedClientResponse mockResponse = + createMockResponse(sw, MediaType.APPLICATION_JSON_TYPE); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(mockResource, mockResponse); + testCase.verify(emptyCollectionsVerifier()); + } + + /** + * Note: does not make sense to check normal jersey Marshaller with not empty + * collections on {@link AppInfo} since + * @XmlJavaTypeAdapter and @XmlPath are altering the marshalling behavior that + * the Jersey Marshaller could not handle! + * @throws Exception + */ + @Test + public void testMarshalNotEmptyCollectionsJson() throws Exception { + JAXBContext eclipseLinkJaxbContext = createEclipseLinkJaxbContext(); + Marshaller jsonMarshaller = createJsonMarshaller(eclipseLinkJaxbContext); + + AppInfo appInfo = createAppInfoWithNotEmptyCollections(); + StringWriter sw = new StringWriter(); + jsonMarshaller.marshal(appInfo, sw); + + WebResource mockResource = createMockResource(); + BufferedClientResponse mockResponse = + createMockResponse(sw, MediaType.APPLICATION_JSON_TYPE); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(mockResource, mockResponse); + testCase.verify(notEmptyCollectionsVerifier()); + } + +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (date 1513958977000) @@ -50,11 +50,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; -import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.*; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -75,6 +71,8 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -90,6 +88,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor; public class TestRMWebServices extends JerseyTestBase { + private static final Logger LOG = LoggerFactory.getLogger(TestRMWebServices.class); private static MockRM rm; @@ -466,19 +465,19 @@ QueueMetrics metrics = rs.getRootQueueMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); - long totalMBExpect = + long totalMBExpect = metrics.getAvailableMB() + metrics.getAllocatedMB(); - long totalVirtualCoresExpect = + long totalVirtualCoresExpect = metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores(); - assertEquals("appsSubmitted doesn't match", + assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), submittedApps); - assertEquals("appsCompleted doesn't match", + assertEquals("appsCompleted doesn't match", metrics.getAppsCompleted(), completedApps); assertEquals("reservedMB doesn't match", metrics.getReservedMB(), reservedMB); - assertEquals("availableMB doesn't match", + assertEquals("availableMB doesn't match", metrics.getAvailableMB(), availableMB); - assertEquals("allocatedMB doesn't match", + assertEquals("allocatedMB doesn't match", metrics.getAllocatedMB(), allocMB); assertEquals("reservedVirtualCores doesn't match", metrics.getReservedVirtualCores(), reservedVirtualCores); @@ -591,11 +590,13 @@ public void verifyClusterSchedulerFifo(JSONObject json) throws JSONException, Exception { - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("incorrect number of elements in: " + json, 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); - assertEquals("incorrect number of elements", 1, info.length()); + assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 11, info.length()); + + LOG.debug("schedulerInfo: {}", info); + assertEquals("incorrect number of elements in: " + info, 11, info.length()); verifyClusterSchedulerFifoGeneric(info.getString("type"), info.getString("qstate"), (float) info.getDouble("capacity"), Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (date 1513958977000) @@ -52,6 +52,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp + .representationhelper.*; +import org.apache.hadoop.yarn.server.resourcemanager.webapp + .representationhelper.json.JsonObjectWrapper; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -79,7 +83,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { private static MockRM rm; - + private static final int CONTAINER_MB = 1024; private static class WebServletModule extends ServletModule { @@ -157,27 +161,27 @@ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").accept(MediaType.APPLICATION_XML) + WebResource path = r.path("ws").path("v1").path("cluster") + .path("apps"); + ClientResponse response = path.accept(MediaType.APPLICATION_XML) .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(xml)); - Document dom = db.parse(is); - NodeList nodesApps = dom.getElementsByTagName("apps"); - assertEquals("incorrect number of elements", 1, nodesApps.getLength()); - NodeList nodes = dom.getElementsByTagName("app"); - assertEquals("incorrect number of elements", 1, nodes.getLength()); - verifyAppsXML(nodes, app1, false); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + ArrayWrapper arrayWrapper = responseAdapter.getArray("apps[]"); + assertEquals("incorrect number of elements", 1, arrayWrapper.length()); + + ArrayWrapper appArray = responseAdapter.getArray("apps.app[]"); + assertEquals("incorrect number of elements", 1, appArray.length()); + verifyAppsXML(appArray, app1, false); + }); + rm.stop(); } @Test - public void testRunningApp() throws JSONException, Exception { + public void testRunningApp() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); @@ -186,22 +190,23 @@ amNodeManager.nodeHeartbeat(true); WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").accept(MediaType.APPLICATION_XML) + WebResource path = r.path("ws").path("v1").path("cluster") + .path("apps"); + ClientResponse response = path.accept(MediaType.APPLICATION_XML) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); - String xml = response.getEntity(String.class); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(xml)); - Document dom = db.parse(is); - NodeList nodesApps = dom.getElementsByTagName("apps"); - assertEquals("incorrect number of elements", 1, nodesApps.getLength()); - NodeList nodes = dom.getElementsByTagName("app"); - assertEquals("incorrect number of elements", 1, nodes.getLength()); - verifyAppsXML(nodes, app1, true); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + ElementWrapper appsWrapper = responseAdapter.getElement("apps"); + assertEquals("incorrect number of elements", 1, appsWrapper.length()); + + ArrayWrapper appArray = appsWrapper.getChildArray("app"); + assertEquals("incorrect number of elements", 1, appArray.length()); + verifyAppsXML(appArray, app1, false); + }); testAppsHelper("apps/", app1, MediaType.APPLICATION_JSON, true); rm.stop(); @@ -254,8 +259,7 @@ assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app, hasResourceReq); - + verifyAppInfoJson(new JsonObjectWrapper(array.getJSONObject(0)), app, hasResourceReq); } @Test @@ -278,7 +282,7 @@ assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app1, false); + verifyAppInfoJson(new JsonObjectWrapper(array.getJSONObject(0)), app1, false); rm.stop(); } @@ -324,7 +328,7 @@ assertEquals("incorrect number of elements", 1, apps.length()); array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 2, array.length()); - assertTrue("both app states of ACCEPTED and KILLED are not present", + assertTrue("both app states of ACCEPTED and KILLED are not present", (array.getJSONObject(0).getString("state").equals("ACCEPTED") && array.getJSONObject(1).getString("state").equals("KILLED")) || (array.getJSONObject(0).getString("state").equals("KILLED") && @@ -375,12 +379,12 @@ assertEquals("incorrect number of elements", 1, apps.length()); array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 2, array.length()); - assertTrue("both app states of ACCEPTED and KILLED are not present", + assertTrue("both app states of ACCEPTED and KILLED are not present", (array.getJSONObject(0).getString("state").equals("ACCEPTED") && array.getJSONObject(1).getString("state").equals("KILLED")) || (array.getJSONObject(0).getString("state").equals("KILLED") && array.getJSONObject(1).getString("state").equals("ACCEPTED"))); - + rm.stop(); } @@ -522,7 +526,7 @@ assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); - verifyAppInfo(array.getJSONObject(0), app1, false); + verifyAppInfoJson(new JsonObjectWrapper(array.getJSONObject(0)), app1, false); rm.stop(); } @@ -1505,7 +1509,7 @@ JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - verifyAppInfo(json.getJSONObject("app"), app, false); + verifyAppInfoJson(new JsonObjectWrapper(json.getJSONObject("app")), app, false); } @Test @@ -1514,270 +1518,81 @@ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").path(app1.getApplicationId().toString()) - .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); + WebResource path = r.path("ws").path("v1").path("cluster") + .path("apps").path(app1.getApplicationId().toString()); + ClientResponse response = path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(xml)); - Document dom = db.parse(is); - NodeList nodes = dom.getElementsByTagName("app"); - assertEquals("incorrect number of elements", 1, nodes.getLength()); - verifyAppsXML(nodes, app1, false); + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + ArrayWrapper arrayWrapper = responseAdapter.getArray("app[]"); + assertEquals("incorrect number of elements", 1, arrayWrapper.length()); + verifyAppsXML(arrayWrapper, app1, false); + }); + rm.stop(); } - public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) - throws JSONException, Exception { - - for (int i = 0; i < nodes.getLength(); i++) { - Element element = (Element) nodes.item(i); - - verifyAppInfoGeneric(app, - WebServicesTestUtils.getXmlString(element, "id"), - WebServicesTestUtils.getXmlString(element, "user"), - WebServicesTestUtils.getXmlString(element, "name"), - WebServicesTestUtils.getXmlString(element, "applicationType"), - WebServicesTestUtils.getXmlString(element, "queue"), - WebServicesTestUtils.getXmlInt(element, "priority"), - WebServicesTestUtils.getXmlString(element, "state"), - WebServicesTestUtils.getXmlString(element, "finalStatus"), - WebServicesTestUtils.getXmlFloat(element, "progress"), - WebServicesTestUtils.getXmlString(element, "trackingUI"), - WebServicesTestUtils.getXmlString(element, "diagnostics"), - WebServicesTestUtils.getXmlLong(element, "clusterId"), - WebServicesTestUtils.getXmlLong(element, "startedTime"), - WebServicesTestUtils.getXmlLong(element, "finishedTime"), - WebServicesTestUtils.getXmlLong(element, "elapsedTime"), - WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"), - WebServicesTestUtils.getXmlString(element, "amContainerLogs"), - WebServicesTestUtils.getXmlInt(element, "allocatedMB"), - WebServicesTestUtils.getXmlInt(element, "allocatedVCores"), - WebServicesTestUtils.getXmlInt(element, "runningContainers"), - WebServicesTestUtils.getXmlFloat(element, "queueUsagePercentage"), - WebServicesTestUtils.getXmlFloat(element, "clusterUsagePercentage"), - WebServicesTestUtils.getXmlInt(element, "preemptedResourceMB"), - WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"), - WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"), - WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"), - WebServicesTestUtils.getXmlString(element, "logAggregationStatus"), - WebServicesTestUtils.getXmlBoolean(element, "unmanagedApplication"), - WebServicesTestUtils.getXmlString(element, "appNodeLabelExpression"), - WebServicesTestUtils.getXmlString(element, "amNodeLabelExpression"), - WebServicesTestUtils.getXmlString(element, "amRPCAddress")); + private void verifyAppsXML(ArrayWrapper array, RMApp app, boolean + hasResourceReq) { + for (int i = 0; i < array.length(); i++) { + ElementWrapper element = array.getObjectAtIndex(i); + AppInfoVerifications.verify(element, app); if (hasResourceReq) { - assertEquals(element.getElementsByTagName("resourceRequests").getLength(), - 1); - Element resourceRequests = - (Element) element.getElementsByTagName("resourceRequests").item(0); - Element capability = - (Element) resourceRequests.getElementsByTagName("capability").item(0); + ArrayWrapper resourceRequests = element.getChildArray("resourceRequests"); + assertEquals(resourceRequests.length(), 1); + ElementWrapper resourceRequest = resourceRequests.getObjectAtIndex(0); ResourceRequest rr = ((AbstractYarnScheduler)rm.getRMContext().getScheduler()) .getApplicationAttempt( app.getCurrentAppAttempt().getAppAttemptId()) .getAppSchedulingInfo().getAllResourceRequests().get(0); - verifyResourceRequestsGeneric(rr, - WebServicesTestUtils.getXmlString(resourceRequests, - "nodeLabelExpression"), - WebServicesTestUtils.getXmlInt(resourceRequests, "numContainers"), - WebServicesTestUtils.getXmlBoolean(resourceRequests, "relaxLocality"), - WebServicesTestUtils.getXmlInt(resourceRequests, "priority"), - WebServicesTestUtils.getXmlString(resourceRequests, "resourceName"), - WebServicesTestUtils.getXmlLong(capability, "memory"), - WebServicesTestUtils.getXmlLong(capability, "vCores"), - WebServicesTestUtils.getXmlString(resourceRequests, "executionType"), - WebServicesTestUtils.getXmlBoolean(resourceRequests, - "enforceExecutionType")); + ResourceRequestsVerifications.verify(resourceRequest, rr); } } } - public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) - throws JSONException, Exception { + private void verifyAppInfoJson(ElementWrapper info, RMApp app, boolean + hasResourceReqs) { + assertEquals("incorrect number of elements", + getExpectedNumberOfElements(app, hasResourceReqs), info.length()); + + AppInfoVerifications.verify(info, app); + if (hasResourceReqs) { + ArrayWrapper resourceRequests = info.getChildArray("resourceRequests"); + ElementWrapper requestInfo = resourceRequests.getObjectAtIndex(0); + ResourceRequest resourceRequest = + ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) + .getApplicationAttempt( + app.getCurrentAppAttempt().getAppAttemptId()) + .getAppSchedulingInfo().getAllResourceRequests().get(0); + + ResourceRequestsVerifications.verify(requestInfo, resourceRequest); + } + } + private int getExpectedNumberOfElements(RMApp app, boolean hasResourceReqs) { int expectedNumberOfElements = 38 + (hasResourceReqs ? 2 : 0); - String appNodeLabelExpression = null; - String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() .getNodeLabelExpression() != null) { expectedNumberOfElements++; - appNodeLabelExpression = info.getString("appNodeLabelExpression"); } if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { expectedNumberOfElements++; - amNodeLabelExpression = info.getString("amNodeLabelExpression"); } - String amRPCAddress = null; + if (AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) { expectedNumberOfElements++; - amRPCAddress = info.getString("amRPCAddress"); - } - assertEquals("incorrect number of elements", expectedNumberOfElements, - info.length()); - verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), - info.getString("name"), info.getString("applicationType"), - info.getString("queue"), info.getInt("priority"), - info.getString("state"), info.getString("finalStatus"), - (float) info.getDouble("progress"), info.getString("trackingUI"), - info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), - info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), - info.getString("amContainerLogs"), info.getInt("allocatedMB"), - info.getInt("allocatedVCores"), info.getInt("runningContainers"), - (float) info.getDouble("queueUsagePercentage"), - (float) info.getDouble("clusterUsagePercentage"), - info.getInt("preemptedResourceMB"), - info.getInt("preemptedResourceVCores"), - info.getInt("numNonAMContainerPreempted"), - info.getInt("numAMContainerPreempted"), - info.getString("logAggregationStatus"), - info.getBoolean("unmanagedApplication"), - appNodeLabelExpression, - amNodeLabelExpression, - amRPCAddress); - - if (hasResourceReqs) { - verifyResourceRequests(info.getJSONArray("resourceRequests"), app); } - } - - public void verifyAppInfoGeneric(RMApp app, String id, String user, - String name, String applicationType, String queue, int prioirty, - String state, String finalStatus, float progress, String trackingUI, - String diagnostics, long clusterId, long startedTime, long finishedTime, - long elapsedTime, String amHostHttpAddress, String amContainerLogs, - int allocatedMB, int allocatedVCores, int numContainers, - float queueUsagePerc, float clusterUsagePerc, - int preemptedResourceMB, int preemptedResourceVCores, - int numNonAMContainerPreempted, int numAMContainerPreempted, - String logAggregationStatus, boolean unmanagedApplication, - String appNodeLabelExpression, String amNodeLabelExpression, - String amRPCAddress) throws JSONException, Exception { - - WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() - .toString(), id); - WebServicesTestUtils.checkStringMatch("user", app.getUser(), user); - WebServicesTestUtils.checkStringMatch("name", app.getName(), name); - WebServicesTestUtils.checkStringMatch("applicationType", - app.getApplicationType(), applicationType); - WebServicesTestUtils.checkStringMatch("queue", app.getQueue(), queue); - assertEquals("priority doesn't match", 0, prioirty); - WebServicesTestUtils.checkStringMatch("state", app.getState().toString(), - state); - WebServicesTestUtils.checkStringMatch("finalStatus", app - .getFinalApplicationStatus().toString(), finalStatus); - assertEquals("progress doesn't match", 0, progress, 0.0); - if ("UNASSIGNED".equals(trackingUI)) { - WebServicesTestUtils.checkStringMatch("trackingUI", "UNASSIGNED", - trackingUI); - } - WebServicesTestUtils.checkStringEqual("diagnostics", - app.getDiagnostics().toString(), diagnostics); - assertEquals("clusterId doesn't match", - ResourceManager.getClusterTimeStamp(), clusterId); - assertEquals("startedTime doesn't match", app.getStartTime(), startedTime); - assertEquals("finishedTime doesn't match", app.getFinishTime(), - finishedTime); - assertTrue("elapsed time not greater than 0", elapsedTime > 0); - WebServicesTestUtils.checkStringMatch("amHostHttpAddress", app - .getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(), - amHostHttpAddress); - assertTrue("amContainerLogs doesn't match", - amContainerLogs.startsWith("http://")); - assertTrue("amContainerLogs doesn't contain user info", - amContainerLogs.endsWith("/" + app.getUser())); - assertEquals("allocatedMB doesn't match", 1024, allocatedMB); - assertEquals("allocatedVCores doesn't match", 1, allocatedVCores); - assertEquals("queueUsagePerc doesn't match", 50.0f, queueUsagePerc, 0.01f); - assertEquals("clusterUsagePerc doesn't match", 50.0f, clusterUsagePerc, 0.01f); - assertEquals("numContainers doesn't match", 1, numContainers); - assertEquals("preemptedResourceMB doesn't match", app - .getRMAppMetrics().getResourcePreempted().getMemorySize(), - preemptedResourceMB); - assertEquals("preemptedResourceVCores doesn't match", app - .getRMAppMetrics().getResourcePreempted().getVirtualCores(), - preemptedResourceVCores); - assertEquals("numNonAMContainerPreempted doesn't match", app - .getRMAppMetrics().getNumNonAMContainersPreempted(), - numNonAMContainerPreempted); - assertEquals("numAMContainerPreempted doesn't match", app - .getRMAppMetrics().getNumAMContainersPreempted(), - numAMContainerPreempted); - assertEquals("Log aggregation Status doesn't match", app - .getLogAggregationStatusForAppReport().toString(), - logAggregationStatus); - assertEquals("unmanagedApplication doesn't match", app - .getApplicationSubmissionContext().getUnmanagedAM(), - unmanagedApplication); - assertEquals("unmanagedApplication doesn't match", - app.getApplicationSubmissionContext().getNodeLabelExpression(), - appNodeLabelExpression); - assertEquals("unmanagedApplication doesn't match", - app.getAMResourceRequests().get(0).getNodeLabelExpression(), - amNodeLabelExpression); - assertEquals("amRPCAddress", - AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()), - amRPCAddress); - } - - public void verifyResourceRequests(JSONArray resourceRequest, RMApp app) - throws JSONException { - JSONObject requestInfo = resourceRequest.getJSONObject(0); - ResourceRequest rr = - ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) - .getApplicationAttempt( - app.getCurrentAppAttempt().getAppAttemptId()) - .getAppSchedulingInfo().getAllResourceRequests().get(0); - verifyResourceRequestsGeneric(rr, - requestInfo.getString("nodeLabelExpression"), - requestInfo.getInt("numContainers"), - requestInfo.getBoolean("relaxLocality"), requestInfo.getInt("priority"), - requestInfo.getString("resourceName"), - requestInfo.getJSONObject("capability").getLong("memory"), - requestInfo.getJSONObject("capability").getLong("vCores"), - requestInfo.getJSONObject("executionTypeRequest") - .getString("executionType"), - requestInfo.getJSONObject("executionTypeRequest") - .getBoolean("enforceExecutionType")); - } - - public void verifyResourceRequestsGeneric(ResourceRequest request, - String nodeLabelExpression, int numContainers, boolean relaxLocality, - int priority, String resourceName, long memory, long vCores, - String executionType, boolean enforceExecutionType) { - assertEquals("nodeLabelExpression doesn't match", - request.getNodeLabelExpression(), nodeLabelExpression); - assertEquals("numContainers doesn't match", request.getNumContainers(), - numContainers); - assertEquals("relaxLocality doesn't match", request.getRelaxLocality(), - relaxLocality); - assertEquals("priority does not match", request.getPriority().getPriority(), - priority); - assertEquals("resourceName does not match", request.getResourceName(), - resourceName); - assertEquals("memory does not match", - request.getCapability().getMemorySize(), memory); - assertEquals("vCores does not match", - request.getCapability().getVirtualCores(), vCores); - assertEquals("executionType does not match", - request.getExecutionTypeRequest().getExecutionType().name(), - executionType); - assertEquals("enforceExecutionType does not match", - request.getExecutionTypeRequest().getEnforceExecutionType(), - enforceExecutionType); + return expectedNumberOfElements; } @Test - public void testAppAttempts() throws JSONException, Exception { + public void testAppAttempts() throws Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java (date 1513958977000) @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.customresourcetypes.CustomResourceTypesConfigurationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.*; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.GuiceServletConfig; +import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.junit.Before; +import org.junit.Test; + +import javax.ws.rs.core.MediaType; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; + +public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase { + + private static MockRM rm; + + private static final int CONTAINER_MB = 1024; + + private static class WebServletModule extends ServletModule { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + Configuration conf = new Configuration(); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, + ResourceScheduler.class); + initResourceTypes(conf); + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + serve("/*").with(GuiceContainer.class); + } + + private void initResourceTypes(Configuration conf) { + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + ResourceUtils.resetResourceTypes(conf); + } + } + + static { + createInjectorForWebServletModule(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + createInjectorForWebServletModule(); + } + + private static void createInjectorForWebServletModule() { + GuiceServletConfig + .setInjector(Guice.createInjector(new WebServletModule())); + } + + public TestRMWebServicesAppsCustomResourceTypes() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testRunningAppXml() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); + am1.allocate("*", 2048, 1, new ArrayList<>()); + amNodeManager.nodeHeartbeat(true); + + WebResource r = resource(); + WebResource path = r.path("ws").path("v1").path("cluster").path("apps"); + ClientResponse response = + path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + ArrayWrapper arrayWrapper = responseAdapter.getArray("apps[]"); + assertEquals("incorrect number of elements", 1, arrayWrapper.length()); + + ArrayWrapper appArray = responseAdapter.getArray("apps.app[]"); + assertEquals("incorrect number of elements", 1, appArray.length()); + + verifyAppsXML(arrayWrapper, app1); + }); + + rm.stop(); + } + + @Test + public void testRunningAppJson() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); + am1.allocate("*", 2048, 1, new ArrayList<>()); + amNodeManager.nodeHeartbeat(true); + + WebResource r = resource(); + WebResource path = r.path("ws").path("v1").path("cluster").path("apps"); + ClientResponse response = + path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(path, + new BufferedClientResponse(response)); + testCase.verify(responseAdapter -> { + assertEquals("incorrect number of elements", 1, + responseAdapter.getElement("apps").length()); + ArrayWrapper appArray = responseAdapter.getArray("apps.app[]"); + assertEquals("incorrect number of elements", 1, appArray.length()); + + verifyAppInfoJson(appArray.getObjectAtIndex(0), app1); + }); + + rm.stop(); + } + + private void verifyAppsXML(ArrayWrapper array, RMApp app) { + for (int i = 0; i < array.length(); i++) { + ElementWrapper element = array.getObjectAtIndex(i); + AppInfoVerifications.verify(element, app); + + ArrayWrapper resourceRequests = element.getChildArray("resourceRequests"); + assertEquals(resourceRequests.length(), 1); + ElementWrapper resourceRequest = resourceRequests.getObjectAtIndex(0); + ResourceRequest rr = + ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) + .getApplicationAttempt( + app.getCurrentAppAttempt().getAppAttemptId()) + .getAppSchedulingInfo().getAllResourceRequests().get(0); + ResourceRequestsVerifications.verifyWithCustomResourceTypes( + resourceRequest, rr, + CustomResourceTypesConfigurationProvider.getCustomResourceTypes()); + } + } + + private void verifyAppInfoJson(ElementWrapper info, RMApp app) { + int expectedNumberOfElements = getExpectedNumberOfElements(app); + + assertEquals("incorrect number of elements", expectedNumberOfElements, + info.length()); + + AppInfoVerifications.verify(info, app); + + ArrayWrapper resourceRequests = info.getChildArray("resourceRequests"); + ElementWrapper requestInfo = resourceRequests.getObjectAtIndex(0); + ResourceRequest rr = + ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) + .getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId()) + .getAppSchedulingInfo().getAllResourceRequests().get(0); + + ResourceRequestsVerifications.verifyWithCustomResourceTypes( + requestInfo, rr, + CustomResourceTypesConfigurationProvider.getCustomResourceTypes()); + } + + private int getExpectedNumberOfElements(RMApp app) { + int expectedNumberOfElements = 38 + 2; // 2 -> resourceRequests + if (app.getApplicationSubmissionContext() + .getNodeLabelExpression() != null) { + expectedNumberOfElements++; + } + + if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { + expectedNumberOfElements++; + } + + if (AppInfo + .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) { + expectedNumberOfElements++; + } + return expectedNumberOfElements; + } + +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java (date 1513958977000) @@ -317,21 +317,21 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException, Exception { - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("incorrect number of elements in: " + json, 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); - assertEquals("incorrect number of elements", 1, info.length()); + assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 8, info.length()); + assertEquals("incorrect number of elements in: " + info, 8, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), (float) info.getDouble("maxCapacity"), info.getString("queueName")); JSONObject health = info.getJSONObject("health"); assertNotNull(health); - assertEquals("incorrect number of elements", 3, health.length()); + assertEquals("incorrect number of elements in: " + health, 3, health.length()); JSONArray arr = info.getJSONObject("queues").getJSONArray("queue"); - assertEquals("incorrect number of elements", 2, arr.length()); + assertEquals("incorrect number of elements in: " + arr, 2, arr.length()); // test subqueues for (int i = 0; i < arr.length(); i++) { Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java (date 1513958977000) @@ -22,6 +22,7 @@ import com.google.inject.servlet.ServletModule; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.container.filter.LoggingFilter; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; import org.apache.hadoop.conf.Configuration; @@ -41,8 +42,10 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.junit.After; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; +import javax.ws.rs.core.Application; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; @@ -51,6 +54,10 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.logging.ConsoleHandler; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -396,6 +403,7 @@ .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); + System.out.println("Response headers: " + response.getHeaders()); assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = cs.getConfiguration(); assertEquals(0.2f, newCSConf Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java (date 1513958977000) @@ -457,7 +457,7 @@ if (object.getClass() == JSONObject.class) { assertEquals("Number of allocations is wrong", 1, realValue); } else if (object.getClass() == JSONArray.class) { - assertEquals("Number of allocations is wrong", + assertEquals("Number of allocations is wrong in: " + object, ((JSONArray) object).length(), realValue); } } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java (date 1515392946000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java (date 1513958977000) @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,13 +16,14 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.resourcemanager.webapp; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler; -import javax.ws.rs.core.MediaType; - +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -30,6 +31,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -38,18 +42,15 @@ import org.codehaus.jettison.json.JSONObject; import org.junit.Before; import org.junit.Test; +import javax.ws.rs.core.MediaType; -import com.google.inject.Guice; -import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class TestRMWebServicesFairScheduler extends JerseyTestBase { private static MockRM rm; private static YarnConfiguration conf; - + private static class WebServletModule extends ServletModule { @Override protected void configureServlets() { @@ -58,7 +59,7 @@ bind(GenericExceptionHandler.class); conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, - ResourceScheduler.class); + ResourceScheduler.class); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); @@ -66,32 +67,32 @@ } static { - GuiceServletConfig.setInjector( - Guice.createInjector(new WebServletModule())); + GuiceServletConfig + .setInjector(Guice.createInjector(new WebServletModule())); } @Before @Override public void setUp() throws Exception { super.setUp(); - GuiceServletConfig.setInjector( - Guice.createInjector(new WebServletModule())); + GuiceServletConfig + .setInjector(Guice.createInjector(new WebServletModule())); } public TestRMWebServicesFairScheduler() { super(new WebAppDescriptor.Builder( "org.apache.hadoop.yarn.server.resourcemanager.webapp") - .contextListenerClass(GuiceServletConfig.class) - .filterClass(com.google.inject.servlet.GuiceFilter.class) - .contextPath("jersey-guice-filter").servletPath("/").build()); + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); } - + @Test - public void testClusterScheduler() throws JSONException, Exception { + public void testClusterScheduler() throws JSONException { WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("scheduler").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + ClientResponse response = + r.path("ws").path("v1").path("cluster").path("scheduler") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); @@ -99,52 +100,51 @@ } @Test - public void testClusterSchedulerSlash() throws JSONException, Exception { + public void testClusterSchedulerSlash() throws JSONException { WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("scheduler/").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + ClientResponse response = + r.path("ws").path("v1").path("cluster").path("scheduler/") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); verifyClusterScheduler(json); } - + @Test - public void testClusterSchedulerWithSubQueues() throws JSONException, - Exception { - FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler(); + public void testClusterSchedulerWithSubQueues() + throws JSONException { + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); QueueManager queueManager = scheduler.getQueueManager(); // create LeafQueue queueManager.getLeafQueue("root.q.subqueue1", true); queueManager.getLeafQueue("root.q.subqueue2", true); WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("scheduler").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + ClientResponse response = + r.path("ws").path("v1").path("cluster").path("scheduler") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); JSONArray subQueueInfo = json.getJSONObject("scheduler") .getJSONObject("schedulerInfo").getJSONObject("rootQueue") - .getJSONObject("childQueues").getJSONArray("queue") - .getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue"); + .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1) + .getJSONObject("childQueues").getJSONArray("queue"); // subQueueInfo is consist of subqueue1 and subqueue2 info assertEquals(2, subQueueInfo.length()); // Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo. try { subQueueInfo.getJSONObject(1).getJSONObject("childQueues"); - fail("FairSchedulerQueueInfo should omit field 'childQueues'" + - "if child queue is empty."); + fail("FairSchedulerQueueInfo should omit field 'childQueues'" + + "if child queue is empty."); } catch (JSONException je) { assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage()); } } - private void verifyClusterScheduler(JSONObject json) throws JSONException, - Exception { + private void verifyClusterScheduler(JSONObject json) throws JSONException { assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/CustomResourceTypesConfigurationProvider.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/CustomResourceTypesConfigurationProvider.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/CustomResourceTypesConfigurationProvider.java (date 1513958977000) @@ -0,0 +1,112 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.customresourcetypes; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.LocalConfigurationProvider; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toList; + +public class CustomResourceTypesConfigurationProvider + extends LocalConfigurationProvider { + + private static class CustomResourceTypes { + private int count; + private String xml; + + CustomResourceTypes(String xml, int count) { + this.xml = xml; + this.count = count; + } + + public int getCount() { + return count; + } + + public String getXml() { + return xml; + } + } + + private static final String CUSTOM_RESOURCE_PREFIX = "customResource-"; + + private static CustomResourceTypes CUSTOM_RESOURCE_TYPES = + createInitialResourceTypes(); + + private static CustomResourceTypes createInitialResourceTypes() { + return createCustomResourceTypes(2); + } + + private static CustomResourceTypes createCustomResourceTypes(int count) { + List resourceTypeNames = generateResourceTypeNames(count); + + List resourceUnitXmlElements = IntStream.range(0, count) + .boxed() + .map(i -> getResourceUnitsXml(resourceTypeNames.get(i))) + .collect(toList()); + + StringBuilder sb = new StringBuilder("\n"); + sb.append(getResourceTypesXml(resourceTypeNames)); + + for (String resourceUnitXml : resourceUnitXmlElements) { + sb.append(resourceUnitXml); + + } + sb.append(""); + + return new CustomResourceTypes(sb.toString(), count); + } + + private static List generateResourceTypeNames(int count) { + return IntStream.range(0, count) + .boxed() + .map(i -> CUSTOM_RESOURCE_PREFIX + i) + .collect(toList()); + } + + private static String getResourceUnitsXml(String resource) { + return "\n" + "yarn.resource-types." + resource + + ".units\n" + "k\n" + "\n"; + } + + private static String getResourceTypesXml(List resources) { + final String resourceTypes = makeCommaSeparatedString(resources); + + return "\n" + "yarn.resource-types\n" + "" + + resourceTypes + "\n" + "\n"; + } + + private static String makeCommaSeparatedString(List resources) { + return resources.stream().collect(Collectors.joining(",")); + } + + @Override + public InputStream getConfigurationInputStream(Configuration bootstrapConf, + String name) throws YarnException, IOException { + if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) { + return new ByteArrayInputStream( + CUSTOM_RESOURCE_TYPES.getXml().getBytes()); + } else { + return super.getConfigurationInputStream(bootstrapConf, name); + } + } + + public static void reset() { + CUSTOM_RESOURCE_TYPES = createInitialResourceTypes(); + } + + public static void setNumberOfResourceTypes(int count) { + CUSTOM_RESOURCE_TYPES = createCustomResourceTypes(count); + } + + public static List getCustomResourceTypes() { + return generateResourceTypeNames(CUSTOM_RESOURCE_TYPES.getCount()); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/FairSchedulerRepresentationVerifications.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/FairSchedulerRepresentationVerifications.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/FairSchedulerRepresentationVerifications.java (date 1513958977000) @@ -0,0 +1,102 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.customresourcetypes; + +import com.google.common.collect.Sets; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; + +import java.util.Set; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class FairSchedulerRepresentationVerifications { + + private static final Set SIMPLE_RESOURCE_CATEGORIES = + Sets.newHashSet("minResources", "amUsedResources", "amMaxResources", + "fairResources", "clusterResources", "reservedResources"); + + private static final Set CUSTOM_RESOURCE_CATEGORIES = + Sets.newHashSet("maxResources", "usedResources", "steadyFairResources", + "demandResources"); + + public void verify(ElementWrapper wrapper) { + verifyResourcesContainDefaultResourceTypes(wrapper, + SIMPLE_RESOURCE_CATEGORIES); + verifyResourcesDoNotContainCustomResourceTypes(wrapper, + SIMPLE_RESOURCE_CATEGORIES); + + verifyResourcesContainsAllResourceTypes(wrapper, + CUSTOM_RESOURCE_CATEGORIES); + } + + private void verifyResourcesContainDefaultResourceTypes(ElementWrapper queue, + Set resourceCategories) { + for (String resourceCategory : resourceCategories) { + boolean hasResourceCategory = queue.hasChild(resourceCategory); + assertTrue("Queue " + queue + " does not have resource category key: " + + resourceCategory, hasResourceCategory); + verifyResourceContainsDefaultResourceTypes( + queue.getChild(resourceCategory)); + } + } + + private void verifyResourceContainsDefaultResourceTypes( + ElementWrapper wrapper) { + Object memory = wrapper.opt("memory"); + Object vCores = wrapper.opt("vCores"); + + assertNotNull("Key 'memory' not found in: " + wrapper, memory); + assertNotNull("Key 'vCores' not found in: " + wrapper, vCores); + } + + private void verifyResourcesDoNotContainCustomResourceTypes( + ElementWrapper firstSubQueue, Set resourceCategories) { + for (String resourceCategory : resourceCategories) { + verifyResourceDoesNotContainCustomResourceTypes( + firstSubQueue.getChild(resourceCategory)); + } + } + + private void verifyResourceDoesNotContainCustomResourceTypes( + ElementWrapper wrapper) { + boolean hasCustomResources = wrapper.hasChild("customResources"); + assertFalse("Resource " + wrapper + " should not have customResources key!", + hasCustomResources); + + for (String resourceType : CustomResourceTypesConfigurationProvider.getCustomResourceTypes()) { + boolean hasResourceType = wrapper.hasChild(resourceType); + assertFalse("Wrapper " + wrapper + + " should not contain custom resource type: " + resourceType, + hasResourceType); + } + } + + private void verifyResourcesContainsAllResourceTypes(ElementWrapper queue, + Set resourceCategories) { + verifyResourcesContainDefaultResourceTypes(queue, resourceCategories); + verifyResourcesContainCustomResourceTypes(queue, resourceCategories); + } + + private void verifyResourcesContainCustomResourceTypes(ElementWrapper queue, + Set resourceCategories) { + for (String resourceCategory : resourceCategories) { + boolean hasResourceCategory = queue.hasChild(resourceCategory); + assertTrue("Queue " + queue + " does not have key for resourceCategory: " + + resourceCategory, hasResourceCategory); + verifyResourceContainsCustomResourceTypes( + queue.getChild(resourceCategory)); + } + } + + private void verifyResourceContainsCustomResourceTypes( + ElementWrapper elementWrapper) { + for (String resourceType : CustomResourceTypesConfigurationProvider.getCustomResourceTypes()) { + boolean hasResourceType = elementWrapper.hasChild(resourceType); + assertTrue( + "ElementWrapper " + elementWrapper + + " does not have expected resource type: " + resourceType, + hasResourceType); + Long resourceTypeValue = elementWrapper.getLong(resourceType); + assertNotNull(resourceTypeValue); + } + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/TestRMWebServicesFairSchedulerCustomResourceTypes.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/TestRMWebServicesFairSchedulerCustomResourceTypes.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/customresourcetypes/TestRMWebServicesFairSchedulerCustomResourceTypes.java (date 1513958977000) @@ -0,0 +1,265 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.customresourcetypes; + +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices; +import org.apache.hadoop.yarn.server.resourcemanager.webapp + .representationhelper.*; + +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.GuiceServletConfig; +import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import javax.ws.rs.core.MediaType; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +public class TestRMWebServicesFairSchedulerCustomResourceTypes + extends JerseyTestBase { + private static MockRM rm; + private static YarnConfiguration conf; + + private static class WebServletModule extends ServletModule { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, + ResourceScheduler.class); + initResourceTypes(conf); + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + serve("/*").with(GuiceContainer.class); + } + + private void initResourceTypes(YarnConfiguration conf) { + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + ResourceUtils.resetResourceTypes(conf); + } + } + + private static class JsonVerifier implements Consumer { + + @Override + public void accept(ResponseAdapter responseAdapter) { + ArrayWrapper arrayWrapper = responseAdapter.getArray( + "scheduler.schedulerInfo.rootQueue.childQueues.queue[1].childQueues.queue[]"); + + // childQueueInfo is consist of subqueue1 and subqueue2 info + assertEquals(2, arrayWrapper.length()); + + ElementWrapper firstChildQueue = arrayWrapper.getObjectAtIndex(0); + new FairSchedulerRepresentationVerifications().verify(firstChildQueue); + } + } + + private static class XmlVerifier implements Consumer { + + @Override + public void accept(ResponseAdapter responseAdapter) { + ArrayWrapper arrayWrapper = responseAdapter.getArray( + "scheduler.schedulerInfo.rootQueue.childQueues.queue[1].childQueues.queue[]"); + assertEquals(2, arrayWrapper.length()); + + ElementWrapper firstChildQueue = arrayWrapper.getObjectAtIndex(0); + new FairSchedulerRepresentationVerifications().verify(firstChildQueue); + } + } + + static { + createInjectorForWebServletModule(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + createInjectorForWebServletModule(); + } + + private static void createInjectorForWebServletModule() { + GuiceServletConfig + .setInjector(Guice.createInjector(new WebServletModule())); + } + + @After + public void teardown() { + CustomResourceTypesConfigurationProvider.reset(); + } + + public TestRMWebServicesFairSchedulerCustomResourceTypes() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testClusterSchedulerWithCustomResourceTypesJson() { + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + QueueManager queueManager = scheduler.getQueueManager(); + // create LeafQueues + queueManager.getLeafQueue("root.q.subqueue1", true); + queueManager.getLeafQueue("root.q.subqueue2", true); + + FSLeafQueue subqueue1 = + queueManager.getLeafQueue("root.q.subqueue1", false); + incrementUsedResourcesOnQueue(subqueue1, 33L); + + WebResource path = + resource().path("ws").path("v1").path("cluster").path("scheduler"); + ClientResponse response = + path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(path, + new BufferedClientResponse(response)); + testCase.verify(new JsonVerifier()); + + ResourceUtils.resetResourceTypes(new Configuration()); + } + + @Test + public void testClusterSchedulerWithCustomResourceTypesXml() { + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + QueueManager queueManager = scheduler.getQueueManager(); + // create LeafQueues + queueManager.getLeafQueue("root.q.subqueue1", true); + queueManager.getLeafQueue("root.q.subqueue2", true); + + FSLeafQueue subqueue1 = + queueManager.getLeafQueue("root.q.subqueue1", false); + incrementUsedResourcesOnQueue(subqueue1, 33L); + + WebResource path = + resource().path("ws").path("v1").path("cluster").path("scheduler"); + ClientResponse response = + path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(new XmlVerifier()); + + ResourceUtils.resetResourceTypes(new Configuration()); + } + + @Test + public void testClusterSchedulerWithElevenCustomResourceTypesXml() { + CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11); + createInjectorForWebServletModule(); + + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + QueueManager queueManager = scheduler.getQueueManager(); + // create LeafQueues + queueManager.getLeafQueue("root.q.subqueue1", true); + queueManager.getLeafQueue("root.q.subqueue2", true); + + FSLeafQueue subqueue1 = + queueManager.getLeafQueue("root.q.subqueue1", false); + incrementUsedResourcesOnQueue(subqueue1, 33L); + + WebResource path = + resource().path("ws").path("v1").path("cluster").path("scheduler"); + ClientResponse response = + path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + + XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase( + path, new BufferedClientResponse(response)); + testCase.verify(new XmlVerifier()); + + ResourceUtils.resetResourceTypes(new Configuration()); + } + + @Test + public void testClusterSchedulerElevenWithCustomResourceTypesJson() { + CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11); + createInjectorForWebServletModule(); + + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + QueueManager queueManager = scheduler.getQueueManager(); + // create LeafQueues + queueManager.getLeafQueue("root.q.subqueue1", true); + queueManager.getLeafQueue("root.q.subqueue2", true); + + FSLeafQueue subqueue1 = + queueManager.getLeafQueue("root.q.subqueue1", false); + incrementUsedResourcesOnQueue(subqueue1, 33L); + + WebResource path = + resource().path("ws").path("v1").path("cluster").path("scheduler"); + ClientResponse response = + path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + JsonCustomResourceTypeTestcase testCase = + new JsonCustomResourceTypeTestcase(path, + new BufferedClientResponse(response)); + testCase.verify(new JsonVerifier()); + + ResourceUtils.resetResourceTypes(new Configuration()); + } + + private void incrementUsedResourcesOnQueue(final FSLeafQueue queue, + final long value) { + try { + Method incUsedResourceMethod = queue.getClass().getSuperclass() + .getDeclaredMethod("incUsedResource", Resource.class); + incUsedResourceMethod.setAccessible(true); + + Map customResources = + CustomResourceTypesConfigurationProvider.getCustomResourceTypes() + .stream() + .collect(Collectors.toMap(Function.identity(), v -> value)); + + incUsedResourceMethod.invoke(queue, + Resource.newInstance(20, 30, customResources)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ArrayWrapper.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ArrayWrapper.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ArrayWrapper.java (date 1513958977000) @@ -0,0 +1,7 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + + +public interface ArrayWrapper extends Wrapper { + int length(); + ElementWrapper getObjectAtIndex(int idx); +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/BufferedClientResponse.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/BufferedClientResponse.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/BufferedClientResponse.java (date 1513958977000) @@ -0,0 +1,32 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + + +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.UniformInterfaceException; + +import javax.ws.rs.core.MediaType; +import java.io.IOException; + +public class BufferedClientResponse { + private ClientResponse response; + + public BufferedClientResponse(ClientResponse response) { + response.bufferEntity(); + this.response = response; + } + + public T getEntity(Class clazz) throws ClientHandlerException, UniformInterfaceException { + try { + response.getEntityInputStream().reset(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return response.getEntity(clazz); + } + + public MediaType getType() { + return response.getType(); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ElementWrapper.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ElementWrapper.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ElementWrapper.java (date 1513958977000) @@ -0,0 +1,19 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +public interface ElementWrapper extends Wrapper { + Object opt(String child); + + Integer getInt(String key); + + Long getLong(String key); + + Float getFloat(String key); + + Double getDouble(String key); + + String getString(String key); + + String getStringSafely(String key); + + Boolean getBoolean(String key); +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/JsonCustomResourceTypeTestcase.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/JsonCustomResourceTypeTestcase.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/JsonCustomResourceTypeTestcase.java (date 1513958977000) @@ -0,0 +1,49 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +import com.sun.jersey.api.client.WebResource; +import org.apache.hadoop.http.JettyUtils; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json.JsonResponseAdapter; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.MediaType; + +import java.util.function.Consumer; + +import static org.junit.Assert.*; + +public class JsonCustomResourceTypeTestcase { + private static final Logger LOG = + LoggerFactory.getLogger(JsonCustomResourceTypeTestcase.class); + + private final WebResource path; + private final BufferedClientResponse response; + private final JSONObject parsedResponse; + + public JsonCustomResourceTypeTestcase(WebResource path, BufferedClientResponse + response) { + this.path = path; + this.response = response; + this.parsedResponse = response.getEntity(JSONObject.class); + } + + public void verify(Consumer verifier) { + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + logResponse(); + + String responseStr = response.getEntity(String.class); + if (responseStr == null || responseStr.isEmpty()) { + throw new IllegalStateException("Response is null or empty!"); + } + ResponseAdapter responseAdapter = new JsonResponseAdapter(response); + verifier.accept(responseAdapter); + } + + private void logResponse() { + LOG.info("Response from service URL {}: {}", path.toString(), parsedResponse); + } +} \ No newline at end of file Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ResponseAdapter.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ResponseAdapter.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/ResponseAdapter.java (date 1513958977000) @@ -0,0 +1,119 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path.ArrayElementPathSegment; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path.Path; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path.PathSegment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertTrue; + +/** + * This class is intended to abstract away the semantics of JSON / XML parser + * libraries, this is done by wrapping the normal elements + * ({@link org.codehaus.jettison.json.JSONObject} / {@link org.w3c.dom.Element} + * and array-typed elements {@link org.codehaus.jettison.json.JSONArray} / + * {@link org.w3c.dom.NodeList}. + * + * The two public methods + * {@link ResponseAdapter#getElement(String)} and + * {@link ResponseAdapter#getArray(String)} are capable of parsing a path that + * describes the hierarchy of the JSON / XML elements. + * + * Example of a path: + * "scheduler.schedulerInfo.rootQueue.childQueues.queue[1].childQueues.queue[]" + * which is the equivalent of this expression with Jettison: + * + *

+ * JSONArray childQueueInfo =
+ *     json.getJSONObject("scheduler")
+ *     .getJSONObject("schedulerInfo")
+ *     .getJSONObject("rootQueue").getJSONObject("childQueues")
+ *     // get the first queue from the array
+ *     .getJSONArray("queue").getJSONObject(1).getJSONObject("childQueues")
+ *     .getJSONArray("queue");
+ * 
+ * + * What this class gives more than the expression above is that it checks whether + * the child element names are found for every parent and it also checks that + * the lengths of the specified arrays contains at least as many elements as the + * queried array element index + 1. + * + * Please note that indexing of the arrays is zero based (developer friendly). + */ +public abstract class ResponseAdapter { + private static final Logger LOG = + LoggerFactory.getLogger(ResponseAdapter.class); + + public ElementWrapper getElement(String pathStr) { + final Path path = Path.create(pathStr); + if (path.isLastSegmentAnArray()) { + throw new IllegalStateException( + "Last segment of part is an array, getArray() should be invoked instead of this method!"); + } + return (ElementWrapper) getElementInternal(createWrapper(), path); + } + + public ArrayWrapper getArray(String pathStr) { + final Path path = Path.create(pathStr); + if (!path.isLastSegmentAnArray()) { + throw new IllegalStateException( + "Last segment of part is a simple element, getElement() should be invoked instead of this method!"); + } + return (ArrayWrapper) getElementInternal(createWrapper(), path); + } + + private Wrapper getElementInternal(Wrapper root, Path path) { + LOG.debug("Path processed so far: '{}'", path.getProcessedPath()); + + final PathSegment pathSegment = path.getNextSegment(); + final String child = pathSegment.getSegmentName(); + + final String processedPath = path.getProcessedPath(); + LOG.debug("Processing element: {}", processedPath); + + final Wrapper wrapper; + if (pathSegment.isArrayElementType()) { + checkChildFound(root, child, processedPath); + + final ArrayWrapper array = root.getChildArray(child); + final int elementIndex = ((ArrayElementPathSegment)pathSegment).getArrayElementIndex(); + checkArrayIndexIsInBounds(array, elementIndex, processedPath); + + wrapper = array.getObjectAtIndex(elementIndex); + } else if (pathSegment.isArrayType()) { + checkChildFound(root, child, processedPath); + wrapper = root.getChildArray(child); + } else { + checkChildFound(root, child, processedPath); + wrapper = root.getChild(child); + } + + if (!path.isLastSegment()) { + return getElementInternal(wrapper, path); + } else { + return wrapper; + } + } + + private void checkArrayIndexIsInBounds(ArrayWrapper array, + int arrayElementIndex, String processedPath) { + final String message = + String.format( + "Array element %s does not have expected number of elements: %d" + + " at path: %s", + array, (arrayElementIndex + 1), processedPath); + assertTrue(message, array.length() >= (arrayElementIndex + 1)); + } + + private void checkChildFound(Wrapper root, String child, + String processedPath) { + final String message = + String.format("Element %s does not have child element: %s at path: %s", + root, child, processedPath); + assertTrue(message, root.hasChild(child)); + } + + public abstract Wrapper createWrapper(); + +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/Wrapper.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/Wrapper.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/Wrapper.java (date 1513958977000) @@ -0,0 +1,8 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +public interface Wrapper { + boolean hasChild(String child); + ElementWrapper getChild(String child); + ArrayWrapper getChildArray(String child); + int length(); +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/XmlCustomResourceTypeTestCase.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/XmlCustomResourceTypeTestCase.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/XmlCustomResourceTypeTestCase.java (date 1513958977000) @@ -0,0 +1,62 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper; + +import com.sun.jersey.api.client.WebResource; +import org.apache.hadoop.http.JettyUtils; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml.XmlResponseAdapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; + +import javax.ws.rs.core.MediaType; +import javax.xml.transform.*; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.StringWriter; +import java.util.function.Consumer; + +import static org.junit.Assert.assertEquals; + +public class XmlCustomResourceTypeTestCase { + private static final Logger LOG = + LoggerFactory.getLogger(XmlCustomResourceTypeTestCase.class); + + private WebResource path; + private BufferedClientResponse response; + + public XmlCustomResourceTypeTestCase(WebResource path, + BufferedClientResponse response) { + this.path = path; + this.response = response; + } + + public void verify(Consumer verifier) { + assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + XmlResponseAdapter responseAdapter = new XmlResponseAdapter(response); + logResponse(responseAdapter.getParsedResponse()); + verifier.accept(responseAdapter); + } + + private void logResponse(Document doc) { + LOG.info("Response from service URL {}: {}", path.toString(), toXml(doc)); + } + + private String toXml(Document doc) { + StringWriter writer; + try { + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty( + "{http://xml.apache.org/xslt}indent" + "-amount", "2"); + writer = new StringWriter(); + transformer.transform(new DOMSource(doc), new StreamResult(writer)); + } catch (TransformerException e) { + throw new RuntimeException(e); + } + + return writer.getBuffer().toString(); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonArrayWrapper.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonArrayWrapper.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonArrayWrapper.java (date 1513958977000) @@ -0,0 +1,49 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ArrayWrapper; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; + + +public class JsonArrayWrapper implements ArrayWrapper { + private JSONArray jsonArray; + + JsonArrayWrapper(JSONArray jsonArray) { + this.jsonArray = jsonArray; + } + + @Override + public int length() { + return jsonArray.length(); + } + + @Override + public ElementWrapper getObjectAtIndex(int idx) { + try { + return new JsonObjectWrapper(jsonArray.getJSONObject(idx)); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean hasChild(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public ElementWrapper getChild(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public ArrayWrapper getChildArray(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return jsonArray.toString(); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonObjectWrapper.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonObjectWrapper.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonObjectWrapper.java (date 1513958977000) @@ -0,0 +1,118 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ArrayWrapper; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +public class JsonObjectWrapper implements ElementWrapper { + private JSONObject jsonObject; + + public JsonObjectWrapper(JSONObject jsonObject) { + this.jsonObject = jsonObject; + } + + @Override + public boolean hasChild(String child) { + return jsonObject.has(child); + } + + @Override + public ElementWrapper getChild(String child) { + try { + return new JsonObjectWrapper(jsonObject.getJSONObject(child)); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public ArrayWrapper getChildArray(String child) { + try { + return new JsonArrayWrapper(jsonObject.getJSONArray(child)); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public int length() { + return jsonObject.length(); + } + + @Override + public Object opt(String child) { + return jsonObject.opt(child); + } + + @Override + public Integer getInt(String key) { + try { + return jsonObject.getInt(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public Float getFloat(String key) { + try { + return (float) jsonObject.getDouble(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public Long getLong(String key) { + try { + return jsonObject.getLong(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public Double getDouble(String key) { + try { + return jsonObject.getDouble(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public String getString(String key) { + try { + return jsonObject.getString(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public String getStringSafely(String key) { + try { + return jsonObject.getString(key); + } catch (JSONException e) { + return null; + } + } + + @Override + public Boolean getBoolean(String key) { + try { + return jsonObject.getBoolean(key); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString() { + if (jsonObject != null) { + return jsonObject.toString(); + } + return null; + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonResponseAdapter.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonResponseAdapter.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/json/JsonResponseAdapter.java (date 1513958977000) @@ -0,0 +1,19 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.json; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.BufferedClientResponse; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ResponseAdapter; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.Wrapper; +import org.codehaus.jettison.json.JSONObject; + +public class JsonResponseAdapter extends ResponseAdapter { + private JSONObject response; + + public JsonResponseAdapter(BufferedClientResponse response) { + this.response = response.getEntity(JSONObject.class); + } + + @Override + public Wrapper createWrapper() { + return new JsonObjectWrapper(response); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayElementPathSegment.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayElementPathSegment.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayElementPathSegment.java (date 1513958977000) @@ -0,0 +1,50 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + +import java.util.regex.Matcher; + +public class ArrayElementPathSegment extends PathSegment { + private final int arrayElementIndex; + + ArrayElementPathSegment(String segmentName, String pathPart) { + super(segmentName); + this.arrayElementIndex = extractArrayElementIndex(pathPart); + } + + private int extractArrayElementIndex(String pathPart) { + int arrayElementIndex = extractIndex(pathPart); + if (arrayElementIndex >= 0) { + return arrayElementIndex; + } else { + throw new IllegalStateException( + "Array arrayElementIndex should be greater than 0!"); + } + } + + private int extractIndex(String pathPart) { + final Matcher matcher = ARRAY_ELEMENT_PATTERN.matcher(pathPart); + + // this call is intentional, without this, matcher groups were empty even if + // matched! + matcher.matches(); + try { + return Integer.parseInt(matcher.group(1)); + } catch (NumberFormatException e) { + throw new IllegalStateException( + "Cannot parse array element index: " + matcher.group()); + } + } + + public int getArrayElementIndex() { + return arrayElementIndex; + } + + @Override + PathSegmentType getType() { + return PathSegmentType.ARRAY_ELEMENT; + } + + @Override + public String toString() { + return String.format("%s[%d]", segmentName, arrayElementIndex); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayPathSegment.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayPathSegment.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/ArrayPathSegment.java (date 1513958977000) @@ -0,0 +1,17 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + +public class ArrayPathSegment extends PathSegment { + ArrayPathSegment(String segmentName) { + super(segmentName); + } + + @Override + PathSegmentType getType() { + return PathSegmentType.ARRAY; + } + + @Override + public String toString() { + return segmentName + "[]"; + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/NormalPathSegment.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/NormalPathSegment.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/NormalPathSegment.java (date 1513958977000) @@ -0,0 +1,18 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + +public class NormalPathSegment extends PathSegment { + + NormalPathSegment(String segmentName) { + super(segmentName); + } + + @Override + PathSegmentType getType() { + return PathSegmentType.NORMAL; + } + + @Override + public String toString() { + return segmentName; + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/Path.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/Path.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/Path.java (date 1513958977000) @@ -0,0 +1,53 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + + +import com.google.common.collect.Lists; + +import java.util.List; +import java.util.stream.Collectors; + +public class Path { + private final List segments; + private int currentIndex; + private final boolean lastSegmentArray; + + public Path(List pathSegments) { + this.segments = pathSegments; + this.currentIndex = 0; + this.lastSegmentArray = + pathSegments.get(pathSegments.size() - 1).isArrayType(); + } + + public static Path create(String path) { + final List pathSegments; + if (path.contains(".")) { + String[] pathParts = path.split("\\."); + + pathSegments = Lists.newArrayList(); + for (String pathPart : pathParts) { + pathSegments.add(PathSegment.create(pathPart)); + } + } else { + pathSegments = Lists.newArrayList(PathSegment.create(path)); + } + return new Path(pathSegments); + } + + public PathSegment getNextSegment() { + return segments.get(currentIndex++); + } + + public boolean isLastSegment() { + return segments.size() == currentIndex; + } + + public boolean isLastSegmentAnArray() { + return lastSegmentArray; + } + + public String getProcessedPath() { + final List subList = segments.subList(0, currentIndex); + return subList.stream().map(PathSegment::toString) + .collect(Collectors.joining(".")); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegment.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegment.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegment.java (date 1513958977000) @@ -0,0 +1,71 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class PathSegment { + + private static final Pattern ARRAY_PATTERN = Pattern.compile(".*\\[\\]"); + static final Pattern ARRAY_ELEMENT_PATTERN = + Pattern.compile(".*\\[(\\d+)\\]"); + + final String segmentName; + + PathSegment(String segmentName) { + this.segmentName = segmentName; + } + + static PathSegment create(final String pathPart) { + final PathSegmentType type = determineType(pathPart); + final String segmentName = extractSegmentName(type, pathPart); + + if (type == PathSegmentType.ARRAY_ELEMENT) { + return new ArrayElementPathSegment(segmentName, pathPart); + } else if (type == PathSegmentType.ARRAY) { + return new ArrayPathSegment(segmentName); + } else { + return new NormalPathSegment(segmentName); + } + } + + private static PathSegmentType determineType(String pathPart) { + final boolean arrayType = ARRAY_PATTERN.matcher(pathPart).matches(); + final Matcher arrayElementMatcher = + ARRAY_ELEMENT_PATTERN.matcher(pathPart); + + if (arrayElementMatcher.matches()) { + return PathSegmentType.ARRAY_ELEMENT; + } else if (arrayType) { + return PathSegmentType.ARRAY; + } else { + return PathSegmentType.NORMAL; + } + } + + private static String extractSegmentName(PathSegmentType type, String pathPart) { + if (type == PathSegmentType.ARRAY_ELEMENT || type == PathSegmentType.ARRAY) { + return extractSegmentName(pathPart); + } else { + return pathPart; + } + } + + private static String extractSegmentName(String pathPart) { + return pathPart.substring(0, pathPart.indexOf('[')); + } + + abstract PathSegmentType getType(); + + public String getSegmentName() { + return segmentName; + } + + public boolean isArrayType() { + return getType() == PathSegmentType.ARRAY; + } + + public boolean isArrayElementType() { + return getType() == PathSegmentType.ARRAY_ELEMENT; + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegmentType.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegmentType.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/path/PathSegmentType.java (date 1513958977000) @@ -0,0 +1,5 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.path; + +public enum PathSegmentType { + NORMAL, ARRAY, ARRAY_ELEMENT +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeListWrapper.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeListWrapper.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeListWrapper.java (date 1513958977000) @@ -0,0 +1,44 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml; + + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ArrayWrapper; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; +import org.w3c.dom.NodeList; + +public class XmlNodeListWrapper implements ArrayWrapper{ + private NodeList nodes; + + XmlNodeListWrapper(NodeList nodes) { + this.nodes = nodes; + } + + @Override + public int length() { + return nodes.getLength(); + } + + @Override + public ElementWrapper getObjectAtIndex(int idx) { + return new XmlNodeWrapper(nodes.item(idx)); + } + + @Override + public boolean hasChild(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public ElementWrapper getChild(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public ArrayWrapper getChildArray(String child) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return nodes.toString(); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeWrapper.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeWrapper.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlNodeWrapper.java (date 1513958977000) @@ -0,0 +1,118 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ArrayWrapper; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ElementWrapper; + + + +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +public class XmlNodeWrapper implements ElementWrapper { + private Node node; + + XmlNodeWrapper(Node node) { + this.node = node; + } + + @Override + public boolean hasChild(String child) { + return getElementsByTagNameInternal(child).getLength() > 0; + } + + @Override + public ElementWrapper getChild(String child) { + Node item = getElementsByTagNameInternal(child).item(0); + return new XmlNodeWrapper(item); + } + + @Override + public ArrayWrapper getChildArray(String child) { + NodeList nodes = getElementsByTagNameInternal(child); + return new XmlNodeListWrapper(nodes); + } + + @Override + public int length() { + // node.getChildNodes().getLength() would return textNodes as well, + // we only need Elements + int count = 0; + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node current = children.item(i); + if (current.getNodeType() == Node.ELEMENT_NODE) { + ++count; + } + } + + return count; + } + + @Override + public Object opt(String child) { + NodeList nodes = getElementsByTagNameInternal(child); + if (nodes.getLength() > 0) { + return nodes.item(0); + } + + return null; + } + + @Override + public Integer getInt(String key) { + return WebServicesTestUtils.getXmlInt((Element) node, key); + } + + @Override + public Float getFloat(String key) { + return WebServicesTestUtils.getXmlFloat((Element) node, key); + } + + @Override + public Long getLong(String key) { + return WebServicesTestUtils.getXmlLong((Element) node, key); + } + + @Override + public Double getDouble(String key) { + return (double) WebServicesTestUtils.getXmlFloat((Element) node, key); + } + + @Override + public String getString(String key) { + return WebServicesTestUtils.getXmlString((Element) node, key); + } + + @Override + public String getStringSafely(String key) { + return getString(key); + } + + @Override + public Boolean getBoolean(String key) { + return WebServicesTestUtils.getXmlBoolean((Element) node, key); + } + + @Override + public String toString() { + if (node != null) { + return node.toString(); + } + return null; + } + + private NodeList getElementsByTagNameInternal(String child) { + if (node instanceof Element) { + return ((Element) node).getElementsByTagName(child); + } else if (node instanceof Document) { + return ((Document) node).getElementsByTagName(child); + } else { + throw new IllegalStateException("Unknown type of wrappedObject: " + + node + ", type: " + node.getClass()); + } + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlResponseAdapter.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlResponseAdapter.java (date 1513958977000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/representationhelper/xml/XmlResponseAdapter.java (date 1513958977000) @@ -0,0 +1,43 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.xml; + + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.BufferedClientResponse; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.ResponseAdapter; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.representationhelper.Wrapper; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.StringReader; + +public class XmlResponseAdapter extends ResponseAdapter { + private final Document parsedResponse; + + public XmlResponseAdapter(BufferedClientResponse response) { + this.parsedResponse = parseXml(response); + } + + private Document parseXml(BufferedClientResponse response) { + try { + String xml = response.getEntity(String.class); + DocumentBuilder db = DocumentBuilderFactory.newInstance() + .newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + + return db.parse(is); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public Document getParsedResponse() { + return parsedResponse; + } + + @Override + public Wrapper createWrapper() { + return new XmlNodeWrapper(parsedResponse); + } +}