resourceSecondsMap) {
+ this.resourceSecondsMap = resourceSecondsMap;
+ }
}
Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (date 1515392946000)
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (date 1513958977000)
@@ -51,12 +51,12 @@
private float fractionMemMaxShare;
private ResourceInfo minResources;
- private ResourceInfo maxResources;
- private ResourceInfo usedResources;
+ private ResourceInfoWithCustomResourceTypes maxResources;
+ private ResourceInfoWithCustomResourceTypes usedResources;
private ResourceInfo amUsedResources;
private ResourceInfo amMaxResources;
- private ResourceInfo demandResources;
- private ResourceInfo steadyFairResources;
+ private ResourceInfoWithCustomResourceTypes demandResources;
+ private ResourceInfoWithCustomResourceTypes steadyFairResources;
private ResourceInfo fairResources;
private ResourceInfo clusterResources;
private ResourceInfo reservedResources;
@@ -88,15 +88,15 @@
amMaxResources = new ResourceInfo(Resource.newInstance(
queue.getMetrics().getMaxAMShareMB(),
queue.getMetrics().getMaxAMShareVCores()));
- usedResources = new ResourceInfo(queue.getResourceUsage());
- demandResources = new ResourceInfo(queue.getDemand());
+ usedResources = new ResourceInfoWithCustomResourceTypes(queue.getResourceUsage());
+ demandResources = new ResourceInfoWithCustomResourceTypes(queue.getDemand());
fractionMemUsed = (float)usedResources.getMemorySize() /
clusterResources.getMemorySize();
- steadyFairResources = new ResourceInfo(queue.getSteadyFairShare());
+ steadyFairResources = new ResourceInfoWithCustomResourceTypes(queue.getSteadyFairShare());
fairResources = new ResourceInfo(queue.getFairShare());
minResources = new ResourceInfo(queue.getMinShare());
- maxResources = new ResourceInfo(
+ maxResources = new ResourceInfoWithCustomResourceTypes(
Resources.componentwiseMin(queue.getMaxShare(),
scheduler.getClusterResource()));
reservedResources = new ResourceInfo(queue.getReservedResource());
@@ -168,7 +168,7 @@
/**
* Returns the steady fair share of this queue in megabytes.
*/
- public ResourceInfo getSteadyFairShare() {
+ public ResourceInfoWithCustomResourceTypes getSteadyFairShare() {
return steadyFairResources;
}
@@ -183,7 +183,7 @@
return minResources;
}
- public ResourceInfo getMaxResources() {
+ public ResourceInfoWithCustomResourceTypes getMaxResources() {
return maxResources;
}
@@ -199,7 +199,7 @@
return queueName;
}
- public ResourceInfo getUsedResources() {
+ public ResourceInfoWithCustomResourceTypes getUsedResources() {
return usedResources;
}
@@ -220,7 +220,7 @@
/**
* @return the demand resource of this queue.
*/
- public ResourceInfo getDemandResources() {
+ public ResourceInfoWithCustomResourceTypes getDemandResources() {
return demandResources;
}
Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfoWithCustomResourceTypes.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfoWithCustomResourceTypes.java (date 1513958977000)
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfoWithCustomResourceTypes.java (date 1513958977000)
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.MapAdapter;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.eclipse.persistence.oxm.annotations.XmlPath;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import java.util.Map;
+import java.util.Set;
+
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.NONE)
+public class ResourceInfoWithCustomResourceTypes {
+ private static final Set DEFAULT_RESOURCE_NAMES =
+ Sets.newHashSet("memory", "vcores");
+
+ @XmlElement
+ long memory;
+ @XmlElement
+ int vCores;
+
+ private Resource resources;
+
+ /**
+ * XmlPath produces a flattened structure, like the keys / values of map were
+ * fields of this object. Example:
+ * "maxResources":{"memory":0,"vCores":0,"customResource1":11,"customResource2":22}
+ */
+ @XmlJavaTypeAdapter(MapAdapter.class)
+ @XmlPath(".")
+ private Map customResources = Maps.newHashMap();
+
+ public ResourceInfoWithCustomResourceTypes() {
+ }
+
+ ResourceInfoWithCustomResourceTypes(final Resource res) {
+ memory = res.getMemorySize();
+ vCores = res.getVirtualCores();
+ resources = Resources.clone(res);
+ customResources = createCustomResources(res);
+ }
+
+ /**
+ *
+ * @param res
+ * @return
+ */
+ private static Map createCustomResources(final Resource res) {
+ final ResourceInformation[] resourceInformations = res.getResources();
+
+ final Map result = Maps.newHashMap();
+ for (ResourceInformation ri : resourceInformations) {
+ final String name = ri.getName();
+
+ if (!isDefaultResource(name)) {
+ result.put(name, ri.getValue());
+ }
+ }
+
+ return result;
+ }
+
+ private static boolean isDefaultResource(final String name) {
+ final String nameLowerCase = name.toLowerCase();
+
+ for (String defaultResourceName : DEFAULT_RESOURCE_NAMES) {
+ if (nameLowerCase.contains(defaultResourceName)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public long getMemorySize() {
+ if (resources == null) {
+ resources = Resource.newInstance(memory, vCores);
+ }
+ return resources.getMemorySize();
+ }
+
+ public int getvCores() {
+ if (resources == null) {
+ resources = Resource.newInstance(memory, vCores);
+ }
+ return resources.getVirtualCores();
+ }
+
+ @Override
+ public String toString() {
+ return resources.toString();
+ }
+
+ public void setMemory(int memory) {
+ if (resources == null) {
+ resources = Resource.newInstance(memory, vCores);
+ }
+ this.memory = memory;
+ resources.setMemorySize(memory);
+ }
+
+ public void setvCores(int vCores) {
+ if (resources == null) {
+ resources = Resource.newInstance(memory, vCores);
+ }
+ this.vCores = vCores;
+ resources.setVirtualCores(vCores);
+ }
+
+ public Resource getResource() {
+ return Resource.newInstance(resources);
+ }
+
+ public Map getCustomResources() {
+ return customResources;
+ }
+
+ public void setCustomResources(Map customResources) {
+ this.customResources = customResources;
+ }
+}
Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java (date 1515392946000)
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java (date 1513958977000)
@@ -39,7 +39,7 @@
@XmlElement(name = "resourceName")
private String resourceName;
@XmlElement(name = "capability")
- private ResourceInfo capability;
+ private ResourceInfoWithCustomResourceTypes capability;
@XmlElement(name = "numContainers")
private int numContainers;
@XmlElement(name = "relaxLocality")
@@ -61,7 +61,7 @@
public ResourceRequestInfo(ResourceRequest request) {
priority = request.getPriority().getPriority();
resourceName = request.getResourceName();
- capability = new ResourceInfo(request.getCapability());
+ capability = new ResourceInfoWithCustomResourceTypes(request.getCapability());
numContainers = request.getNumContainers();
relaxLocality = request.getRelaxLocality();
nodeLabelExpression = request.getNodeLabelExpression();
@@ -87,11 +87,11 @@
this.resourceName = resourceName;
}
- public ResourceInfo getCapability() {
+ public ResourceInfoWithCustomResourceTypes getCapability() {
return capability;
}
- public void setCapability(ResourceInfo capability) {
+ public void setCapability(ResourceInfoWithCustomResourceTypes capability) {
this.capability = capability;
}
Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java (date 1515392946000)
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java (date 1513958977000)
@@ -23,6 +23,7 @@
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
+import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -74,9 +75,13 @@
}
public String getSchedulerResourceTypes() {
- return Arrays.toString(minAllocResource.getResource().getResources());
+ if (minAllocResource != null) {
+ return Arrays.toString(minAllocResource.getResource().getResources());
+ }
+ return null;
}
+ @XmlTransient
public int getMaxClusterLevelAppPriority() {
return this.maximumClusterPriority;
}
Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/JSONRootElementProviderEclipseLink.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/JSONRootElementProviderEclipseLink.java (date 1513958977000)
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/JSONRootElementProviderEclipseLink.java (date 1513958977000)
@@ -0,0 +1,233 @@
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper;
+
+import com.google.common.collect.ImmutableSet;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.core.header.QualitySourceMediaType;
+import com.sun.jersey.core.provider.jaxb.AbstractRootElementProvider;
+import com.sun.jersey.core.util.FeaturesAndProperties;
+import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.ext.Providers;
+import javax.xml.bind.*;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver.ECLIPSELINK_SERIALIZED_TYPES;
+
+/**
+ *
+ * This class is an extension of {@link JSONRootElementProvider} in a sense that
+ * it replaces the functionality of the
+ * {@link JSONRootElementProvider#writeTo(Object, MediaType, Charset, Marshaller, OutputStream)}
+ * with a custom implementation.
+ *
+ * The reason why the original
+ * {@link JSONRootElementProvider#writeTo(Object, MediaType, Charset, Marshaller, OutputStream)}
+ * is not adequate is that it calls
+ * {@link JSONJAXBContext#getJSONMarshaller(Marshaller, JAXBContext)} that
+ * creates a {@link com.sun.jersey.json.impl.BaseJSONMarshaller} in case of the
+ * marshaller is not an instance of {@link JSONMarshaller}.
+ *
+ * Currently there are some POJOs, e.g. {@link SchedulerTypeInfo},
+ * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo} that
+ * are serialized with EclipseLink's JAXB implementation, see implementation in
+ * {@link JAXBContextResolver#createTypesContextMap()}. If a class with
+ * EclipseLink JAXB is being serialized, the marshaller created by Jersey is not
+ * an instance of {@link JSONMarshaller} therefore
+ * {@link com.sun.jersey.api.json.JSONConfiguration#DEFAULT} would be used as a
+ * JSON config for serialization.
+ *
+ * Since the {@link com.sun.jersey.api.json.JSONConfiguration#DEFAULT} is a
+ * hardcoded JSON config and it turns on root unwrapping, therefore the root tag
+ * would not have been serialized.
+ *
+ * A more simplistic approach would have been to replace
+ * {@link com.sun.jersey.api.json.JSONConfiguration#DEFAULT} with a more
+ * feasible configuration (without root unwrapping) but it caused several tests
+ * to fail since some webservice relied on the default configuration.
+ *
+ * Every operation except
+ * {@link JSONRootElementProviderEclipseLink#writeTo(Object, MediaType, Charset, Marshaller, OutputStream)}
+ * is delegated to {@link JSONRootElementProvider} methods, respectively.
+ * Subclassing the delegate is not possible as constructors are package-private.
+ *
+ * The only special thing is in
+ * {@link App#isWriteable(Class, Type, Annotation[], MediaType)} that only
+ * treats a passed-in type as supported if it is included in the
+ * {@link App#SUPPORTED_TYPES} set. This way, we can limit the serialization
+ * representation's scope to such supported classes. Delegate method
+ * {@link JSONRootElementProviderEclipseLink#readFrom(Class, MediaType, Unmarshaller, InputStream)}
+ * is treated special since this method is protected on the delagate so the call
+ * should utilize reflection.
+ *
+ */
+public class JSONRootElementProviderEclipseLink
+ extends AbstractRootElementProvider {
+ private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(JSONRootElementProviderEclipseLink.class);
+
+ private final JSONRootElementProvider delegate;
+
+ JSONRootElementProviderEclipseLink(Providers ps) throws Exception {
+ super(ps);
+ this.delegate = createDelegateInstance(ps);
+ }
+
+ JSONRootElementProviderEclipseLink(Providers ps, MediaType mt)
+ throws Exception {
+ super(ps, mt);
+ this.delegate = createDelegateInstance(ps, mt);
+ }
+
+ /**
+ * Since JSONRootElementProvider constructors are package-private, it cannot
+ * be instantiated simply, but with reflection.
+ *
+ * @param providers
+ * @return
+ */
+ private static JSONRootElementProvider createDelegateInstance(
+ Providers providers) throws Exception {
+
+ Constructor constructor =
+ JSONRootElementProvider.class.getDeclaredConstructor(Providers.class);
+ constructor.setAccessible(true);
+ return constructor.newInstance(providers);
+ }
+
+ /**
+ * Since JSONRootElementProvider constructors are package-private, it cannot
+ * be instantiated simply, but with reflection.
+ *
+ * @param providers
+ * @return
+ */
+ private static JSONRootElementProvider createDelegateInstance(
+ Providers providers, MediaType mediaType) throws Exception {
+
+ Constructor constructor =
+ JSONRootElementProvider.class.getDeclaredConstructor(Providers.class,
+ MediaType.class);
+ constructor.setAccessible(true);
+ return constructor.newInstance(providers, mediaType);
+ }
+
+ @Context
+ @Override
+ public void setConfiguration(FeaturesAndProperties fp) {
+ delegate.setConfiguration(fp);
+ }
+
+ @Override
+ public boolean isReadable(Class> type, Type genericType,
+ Annotation[] annotations, MediaType mediaType) {
+ return delegate.isReadable(type, genericType, annotations, mediaType);
+ }
+
+ @Override
+ public boolean isWriteable(Class> type, Type genericType,
+ Annotation[] annotations, MediaType mediaType) {
+ return delegate.isWriteable(type, genericType, annotations, mediaType);
+ }
+
+ @Override
+ protected final Object readFrom(Class