diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java index 9dc0c1d..734c741 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java @@ -20,16 +20,17 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "appattempt") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class ApplicationAttemptEntity extends HierarchicalTimelineEntity { public ApplicationAttemptEntity() { super(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString()); } + + public ApplicationAttemptEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java index 45ec520..df1281e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java @@ -20,28 +20,27 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "application") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class ApplicationEntity extends HierarchicalTimelineEntity { - private String queue; + public static final String QUEUE_INFO_KEY = "QUEUE_INFO"; public ApplicationEntity() { super(TimelineEntityType.YARN_APPLICATION.toString()); } - @XmlElement(name = "queue") + public ApplicationEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } + public String getQueue() { - return queue; + return getInfo().get(QUEUE_INFO_KEY).toString(); } public void setQueue(String queue) { - this.queue = queue; + addInfo(QUEUE_INFO_KEY, queue); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java index a4278c0..94eefa8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java @@ -20,12 +20,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "cluster") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class ClusterEntity extends HierarchicalTimelineEntity { @@ -33,4 +27,10 @@ public ClusterEntity() { super(TimelineEntityType.YARN_CLUSTER.toString()); } + public ClusterEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_CLUSTER.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java index cde6040..c7d6bce 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java @@ -20,16 +20,17 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "container") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class ContainerEntity extends HierarchicalTimelineEntity { public ContainerEntity() { super(TimelineEntityType.YARN_CONTAINER.toString()); } + + public ContainerEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_CONTAINER.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java index 0765f00..a66982f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java @@ -20,62 +20,80 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -@XmlRootElement(name = "flow") -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class FlowEntity extends HierarchicalTimelineEntity { - private String user; - private String version; - private String run; + public static final String USER_INFO_KEY = "USER_INFO"; + public static final String FLOW_NAME_INFO_KEY = "FLOW_NAME_INFO"; + public static final String FLOW_VERSION_INFO_KEY = "FLOW_VERSION_INFO"; + public static final String FLOW_RUN_ID_INFO_KEY = "FLOW_RUN_ID_INFO"; public FlowEntity() { super(TimelineEntityType.YARN_FLOW.toString()); } + public FlowEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_FLOW.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } + + @XmlElement(name = "id") @Override public String getId() { - //Flow id schema: user@flow_name(or id)/version/run - StringBuilder sb = new StringBuilder(); - sb.append(user); - sb.append('@'); - sb.append(super.getId()); - sb.append('/'); - sb.append(version); - sb.append('/'); - sb.append(run); - return sb.toString(); + //Flow id schema: user@flow_name(or id)/version/run_id + String id = super.getId(); + if (id == null) { + StringBuilder sb = new StringBuilder(); + sb.append(getInfo().get(USER_INFO_KEY).toString()); + sb.append('@'); + sb.append(getInfo().get(FLOW_NAME_INFO_KEY).toString()); + sb.append('/'); + sb.append(getInfo().get(FLOW_VERSION_INFO_KEY).toString()); + sb.append('/'); + sb.append(getInfo().get(FLOW_RUN_ID_INFO_KEY).toString()); + return sb.toString(); + } else { + return id; + } } - @XmlElement(name = "user") public String getUser() { - return user; + Object user = getInfo().get(USER_INFO_KEY); + return user == null ? null : user.toString(); } public void setUser(String user) { - this.user = user; + addInfo(USER_INFO_KEY, user); + } + + public String getName() { + Object name = getInfo().get(FLOW_NAME_INFO_KEY); + return name == null ? null : name.toString(); + } + + public void setName(String name) { + addInfo(FLOW_NAME_INFO_KEY, name); } - @XmlElement(name = "version") public String getVersion() { - return version; + Object version = getInfo().get(FLOW_VERSION_INFO_KEY); + return version == null ? null : version.toString(); } public void setVersion(String version) { - this.version = version; + addInfo(FLOW_VERSION_INFO_KEY, version); } - @XmlElement(name = "run") - public String getRun() { - return run; + public long getRunId() { + Object runId = getInfo().get(FLOW_RUN_ID_INFO_KEY); + return runId == null ? 0L : new Long((Long) runId); } - public void setRun(String run) { - this.run = run; + public void setRunId(long runId) { + addInfo(FLOW_RUN_ID_INFO_KEY, runId); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java index 49576de..e4af288 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java @@ -20,84 +20,77 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -@XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public abstract class HierarchicalTimelineEntity extends TimelineEntity { - private Identifier parent; - private HashMap> children = new HashMap<>(); + HierarchicalTimelineEntity(TimelineEntity entity) { + super(entity); + } HierarchicalTimelineEntity(String type) { super(type); } - @XmlElement(name = "parent") public Identifier getParent() { - return parent; + Set keys = getIsRelatedToEntities().keySet(); + if (keys.size() == 0) { + return null; + } else if (keys.size() == 1) { + String key = keys.iterator().next(); + validateParent(key); + Set values = getIsRelatedToEntities().get(key); + if (values.size() == 0) { + return null; + } else if (values.size() == 1) { + String value = values.iterator().next(); + Identifier identifier = new Identifier(); + identifier.setType(key); + identifier.setId(value); + return identifier; + } + } + throw new IllegalArgumentException(this.getIdentifier() + + " has more than one parents."); } public void setParent(Identifier parent) { validateParent(parent.getType()); - this.parent = parent; + addIsRelatedToEntity(parent.getType(), parent.getId()); } public void setParent(String type, String id) { validateParent(type); - parent = new Identifier(); - parent.setType(type); - parent.setId(id); - } - - // required by JAXB - @InterfaceAudience.Private - // comment out XmlElement here because it cause UnrecognizedPropertyException - // TODO we need a better fix - //@XmlElement(name = "children") - public HashMap> getChildrenJAXB() { - return children; + addIsRelatedToEntity(type, id); } public Map> getChildren() { + Map> children = getRelatesToEntities(); + validateChildren(children); return children; } public void setChildren(Map> children) { validateChildren(children); - if (children != null && !(children instanceof HashMap)) { - this.children = new HashMap>(children); - } else { - this.children = (HashMap) children; - } + setRelatesToEntities(children); } public void addChildren(Map> children) { validateChildren(children); - for (Map.Entry> entry : children.entrySet()) { - Set ids = this.children.get(entry.getKey()); - if (ids == null) { - ids = new HashSet<>(); - this.children.put(entry.getKey(), ids); - } - ids.addAll(entry.getValue()); - } + addRelatesToEntities(children); } public void addChild(String type, String id) { TimelineEntityType thisType = TimelineEntityType.valueOf(getType()); TimelineEntityType childType = TimelineEntityType.valueOf(type); if (thisType.isChild(childType)) { - Set ids = children.get(type); + Set ids = getRelatesToEntities().get(type); if (ids == null) { ids = new HashSet<>(); - children.put(type, ids); + getRelatesToEntities().put(type, ids); } ids.add(id); } else { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java new file mode 100644 index 0000000..d1a1f19 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records.timelineservice; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class QueueEntity extends HierarchicalTimelineEntity { + public QueueEntity() { + super(TimelineEntityType.YARN_QUEUE.toString()); + } + + public QueueEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_QUEUE.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java index 1afb564..fec4c5c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java @@ -62,8 +62,16 @@ public String getId() { public void setId(String id) { this.id = id; } + + @Override + public String toString() { + return "TimelineEntity[" + + "type='" + type + '\'' + + ", id='" + id + '\'' + "]"; + } } + private TimelineEntity prototype; private Identifier identifier; private HashMap info = new HashMap<>(); private HashMap configs = new HashMap<>(); @@ -77,6 +85,9 @@ public void setId(String id) { public TimelineEntity() { identifier = new Identifier(); } + public TimelineEntity(TimelineEntity entity) { + prototype = entity.getPrototype(); + } protected TimelineEntity(String type) { this(); @@ -85,82 +96,146 @@ protected TimelineEntity(String type) { @XmlElement(name = "type") public String getType() { - return identifier.type; + if (prototype == null) { + return identifier.type; + } else { + return prototype.getType(); + } } public void setType(String type) { - identifier.type = type; + if (prototype == null) { + identifier.type = type; + } else { + prototype.setType(type); + } } @XmlElement(name = "id") public String getId() { - return identifier.id; + if (prototype == null) { + return identifier.id; + } else { + return prototype.getId(); + } } public void setId(String id) { - identifier.id = id; + if (prototype == null) { + identifier.id = id; + } else { + prototype.setId(id); + } } public Identifier getIdentifier() { - return identifier; + if (prototype == null) { + return identifier; + } else { + return prototype.getIdentifier(); + } } public void setIdentifier(Identifier identifier) { - this.identifier = identifier; + if (prototype == null) { + this.identifier = identifier; + } else { + prototype.setIdentifier(identifier); + } } // required by JAXB @InterfaceAudience.Private @XmlElement(name = "info") public HashMap getInfoJAXB() { - return info; + if (prototype == null) { + return info; + } else { + return prototype.getInfoJAXB(); + } } public Map getInfo() { - return info; + if (prototype == null) { + return info; + } else { + return prototype.getInfo(); + } } public void setInfo(Map info) { - if (info != null && !(info instanceof HashMap)) { - this.info = new HashMap(info); + if (prototype == null) { + if (info != null && !(info instanceof HashMap)) { + this.info = new HashMap(info); + } else { + this.info = (HashMap) info; + } } else { - this.info = (HashMap) info; + prototype.setInfo(info); } } public void addInfo(Map info) { - this.info.putAll(info); + if (prototype == null) { + this.info.putAll(info); + } else { + prototype.addInfo(info); + } } public void addInfo(String key, Object value) { - info.put(key, value); + if (prototype == null) { + info.put(key, value); + } else { + prototype.addInfo(key, value); + } } // required by JAXB @InterfaceAudience.Private @XmlElement(name = "configs") public HashMap getConfigsJAXB() { - return configs; + if (prototype == null) { + return configs; + } else { + return prototype.getConfigsJAXB(); + } } public Map getConfigs() { - return configs; + if (prototype == null) { + return configs; + } else { + return prototype.getConfigs(); + } } public void setConfigs(Map configs) { - if (configs != null && !(configs instanceof HashMap)) { - this.configs = new HashMap(configs); + if (prototype == null) { + if (configs != null && !(configs instanceof HashMap)) { + this.configs = new HashMap(configs); + } else { + this.configs = (HashMap) configs; + } } else { - this.configs = (HashMap) configs; + prototype.setConfigs(configs); } } public void addConfigs(Map configs) { - this.configs.putAll(configs); + if (prototype == null) { + this.configs.putAll(configs); + } else { + prototype.addConfigs(configs); + } } public void addConfig(String key, Object value) { - configs.put(key, value); + if (prototype == null) { + configs.put(key, value); + } else { + prototype.addConfig(key, value); + } } @XmlElement(name = "metrics") @@ -169,11 +244,19 @@ public void addConfig(String key, Object value) { } public void setMetrics(Set metrics) { - this.metrics = metrics; + if (prototype == null) { + this.metrics = metrics; + } else { + prototype.setMetrics(metrics); + } } public void addMetrics(Set metrics) { - this.metrics.addAll(metrics); + if (prototype == null) { + this.metrics.addAll(metrics); + } else { + prototype.addMetrics(metrics); + } } public void addMetric(TimelineMetric metric) { @@ -182,119 +265,201 @@ public void addMetric(TimelineMetric metric) { @XmlElement(name = "events") public Set getEvents() { - return events; + if (prototype == null) { + return events; + } else { + return prototype.getEvents(); + } } public void setEvents(Set events) { - this.events = events; + if (prototype == null) { + this.events = events; + } else { + prototype.setEvents(events); + } } public void addEvents(Set events) { - this.events.addAll(events); + if (prototype == null) { + this.events.addAll(events); + } else { + prototype.addEvents(events); + } } public void addEvent(TimelineEvent event) { - events.add(event); + if (prototype == null) { + events.add(event); + } else { + prototype.addEvent(event); + } } public Map> getIsRelatedToEntities() { - return isRelatedToEntities; + if (prototype == null) { + return isRelatedToEntities; + } else { + return prototype.getIsRelatedToEntities(); + } } // required by JAXB @InterfaceAudience.Private @XmlElement(name = "isrelatedto") public HashMap> getIsRelatedToEntitiesJAXB() { - return isRelatedToEntities; + if (prototype == null) { + return isRelatedToEntities; + } else { + return prototype.getIsRelatedToEntitiesJAXB(); + } } public void setIsRelatedToEntities( Map> isRelatedToEntities) { - if (isRelatedToEntities != null && !(isRelatedToEntities instanceof HashMap)) { - this.isRelatedToEntities = new HashMap>(isRelatedToEntities); + if (prototype == null) { + if (isRelatedToEntities != null && + !(isRelatedToEntities instanceof HashMap)) { + this.isRelatedToEntities = + new HashMap>(isRelatedToEntities); + } else { + this.isRelatedToEntities = + (HashMap>) isRelatedToEntities; + } } else { - this.isRelatedToEntities = (HashMap>) isRelatedToEntities; + prototype.setIsRelatedToEntities(isRelatedToEntities); } } public void addIsRelatedToEntities( Map> isRelatedToEntities) { - for (Map.Entry> entry : isRelatedToEntities - .entrySet()) { - Set ids = this.isRelatedToEntities.get(entry.getKey()); - if (ids == null) { - ids = new HashSet<>(); - this.isRelatedToEntities.put(entry.getKey(), ids); + if (prototype == null) { + for (Map.Entry> entry : isRelatedToEntities + .entrySet()) { + Set ids = this.isRelatedToEntities.get(entry.getKey()); + if (ids == null) { + ids = new HashSet<>(); + this.isRelatedToEntities.put(entry.getKey(), ids); + } + ids.addAll(entry.getValue()); } - ids.addAll(entry.getValue()); + } else { + prototype.addIsRelatedToEntities(isRelatedToEntities); } } public void addIsRelatedToEntity(String type, String id) { - Set ids = isRelatedToEntities.get(type); - if (ids == null) { - ids = new HashSet<>(); - isRelatedToEntities.put(type, ids); + if (prototype == null) { + Set ids = isRelatedToEntities.get(type); + if (ids == null) { + ids = new HashSet<>(); + isRelatedToEntities.put(type, ids); + } + ids.add(id); + } else { + prototype.addIsRelatedToEntity(type, id); } - ids.add(id); } // required by JAXB @InterfaceAudience.Private @XmlElement(name = "relatesto") public HashMap> getRelatesToEntitiesJAXB() { - return relatesToEntities; + if (prototype == null) { + return relatesToEntities; + } else { + return prototype.getRelatesToEntitiesJAXB(); + } } public Map> getRelatesToEntities() { - return relatesToEntities; + if (prototype == null) { + return relatesToEntities; + } else { + return prototype.getRelatesToEntities(); + } } public void addRelatesToEntities(Map> relatesToEntities) { - for (Map.Entry> entry : relatesToEntities.entrySet()) { - Set ids = this.relatesToEntities.get(entry.getKey()); - if (ids == null) { - ids = new HashSet<>(); - this.relatesToEntities.put(entry.getKey(), ids); + if (prototype == null) { + for (Map.Entry> entry : relatesToEntities + .entrySet()) { + Set ids = this.relatesToEntities.get(entry.getKey()); + if (ids == null) { + ids = new HashSet<>(); + this.relatesToEntities.put(entry.getKey(), ids); + } + ids.addAll(entry.getValue()); } - ids.addAll(entry.getValue()); + } else { + prototype.addRelatesToEntities(relatesToEntities); } } public void addRelatesToEntity(String type, String id) { - Set ids = relatesToEntities.get(type); - if (ids == null) { - ids = new HashSet<>(); - relatesToEntities.put(type, ids); + if (prototype == null) { + Set ids = relatesToEntities.get(type); + if (ids == null) { + ids = new HashSet<>(); + relatesToEntities.put(type, ids); + } + ids.add(id); + } else { + prototype.addRelatesToEntity(type, id); } - ids.add(id); } public void setRelatesToEntities(Map> relatesToEntities) { - if (relatesToEntities != null && !(relatesToEntities instanceof HashMap)) { - this.relatesToEntities = new HashMap>(relatesToEntities); + if (prototype == null) { + if (relatesToEntities != null && + !(relatesToEntities instanceof HashMap)) { + this.relatesToEntities = + new HashMap>(relatesToEntities); + } else { + this.relatesToEntities = + (HashMap>) relatesToEntities; + } } else { - this.relatesToEntities = (HashMap>) relatesToEntities; + prototype.setRelatesToEntities(relatesToEntities); } } @XmlElement(name = "createdtime") public long getCreatedTime() { - return createdTime; + if (prototype == null) { + return createdTime; + } else { + return prototype.getCreatedTime(); + } } public void setCreatedTime(long createdTime) { - this.createdTime = createdTime; + if (prototype == null) { + this.createdTime = createdTime; + } else { + prototype.setCreatedTime(createdTime); + } } @XmlElement(name = "modifiedtime") public long getModifiedTime() { - return modifiedTime; + if (prototype == null) { + return modifiedTime; + } else { + return prototype.getModifiedTime(); + } } public void setModifiedTime(long modifiedTime) { - this.modifiedTime = modifiedTime; + if (prototype == null) { + this.modifiedTime = modifiedTime; + } else { + prototype.setModifiedTime(modifiedTime); + } } + protected TimelineEntity getPrototype() { + return prototype == null ? this : prototype; + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java deleted file mode 100644 index c4039f8..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.api.records.timelineservice; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "queue") -@XmlAccessorType(XmlAccessType.NONE) -@InterfaceAudience.Public -@InterfaceStability.Unstable -public class TimelineQueue extends HierarchicalTimelineEntity { - public TimelineQueue() { - super(TimelineEntityType.YARN_QUEUE.toString()); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java deleted file mode 100644 index 45cf48f..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.api.records.timelineservice; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement(name = "user") -@XmlAccessorType(XmlAccessType.NONE) -@InterfaceAudience.Public -@InterfaceStability.Unstable -public class TimelineUser extends TimelineEntity { - public TimelineUser() { - super(TimelineEntityType.YARN_USER.toString()); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java new file mode 100644 index 0000000..a229fd8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records.timelineservice; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class UserEntity extends TimelineEntity { + public UserEntity() { + super(TimelineEntityType.YARN_USER.toString()); + } + + public UserEntity(TimelineEntity entity) { + super(entity); + if (!entity.getType().equals(TimelineEntityType.YARN_USER.toString())) { + throw new IllegalArgumentException("Incompatible entity type: " + getId()); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java index 4f8ab94..3ad7433 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java @@ -87,10 +87,10 @@ public void testTimelineEntities() throws Exception { @Test public void testFirstClassCitizenEntities() throws Exception { - TimelineUser user = new TimelineUser(); + UserEntity user = new UserEntity(); user.setId("test user id"); - TimelineQueue queue = new TimelineQueue(); + QueueEntity queue = new QueueEntity(); queue.setId("test queue id"); @@ -98,16 +98,18 @@ public void testFirstClassCitizenEntities() throws Exception { cluster.setId("test cluster id"); FlowEntity flow1 = new FlowEntity(); - flow1.setId("test flow id"); + //flow1.setId("test flow id 1"); flow1.setUser(user.getId()); - flow1.setVersion("test flow version"); - flow1.setRun("test run 1"); + flow1.setName("test flow name 1"); + flow1.setVersion("test flow version 1"); + flow1.setRunId(1L); FlowEntity flow2 = new FlowEntity(); - flow2.setId("test flow run id2"); + //flow2.setId("test flow run id 2"); flow2.setUser(user.getId()); - flow1.setVersion("test flow version2"); - flow2.setRun("test run 2"); + flow2.setName("test flow name 2"); + flow2.setVersion("test flow version 2"); + flow2.setRunId(2L); ApplicationEntity app = new ApplicationEntity(); app.setId(ApplicationId.newInstance(0, 1).toString()); @@ -148,7 +150,7 @@ public void testFirstClassCitizenEntities() throws Exception { @Test public void testUser() throws Exception { - TimelineUser user = new TimelineUser(); + UserEntity user = new UserEntity(); user.setId("test user id"); user.addInfo("test info key 1", "test info value 1"); user.addInfo("test info key 2", "test info value 2"); @@ -157,7 +159,7 @@ public void testUser() throws Exception { @Test public void testQueue() throws Exception { - TimelineQueue queue = new TimelineQueue(); + QueueEntity queue = new QueueEntity(); queue.setId("test queue id"); queue.addInfo("test info key 1", "test info value 1"); queue.addInfo("test info key 2", "test info value 2"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java index c8b9625..93ab230 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java @@ -24,9 +24,12 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.timelineservice.*; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -35,6 +38,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse; import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService; import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager; +import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -85,6 +89,43 @@ public void testPutEntities() throws Exception { } } + @Test + public void testPutExtendedEntities() throws Exception { + ApplicationId appId = ApplicationId.newInstance(0, 1); + TimelineClient client = + TimelineClient.createTimelineClient(appId); + try { + // set the timeline service address manually + client.setTimelineServiceAddress( + collectorManager.getRestServerBindAddress()); + client.init(new YarnConfiguration()); + client.start(); + ClusterEntity cluster = new ClusterEntity(); + cluster.setId(YarnConfiguration.DEFAULT_RM_CLUSTER_ID); + FlowEntity flow = new FlowEntity(); + flow.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); + flow.setName("test_flow_name"); + flow.setVersion("test_flow_version"); + flow.setRunId(1L); + ApplicationEntity app = new ApplicationEntity(); + app.setId(appId.toString()); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); + ApplicationAttemptEntity appAttempt = new ApplicationAttemptEntity(); + appAttempt.setId(attemptId.toString()); + ContainerId containerId = ContainerId.newContainerId(attemptId, 1); + ContainerEntity container = new ContainerEntity(); + container.setId(containerId.toString()); + UserEntity user = new UserEntity(); + user.setId(UserGroupInformation.getCurrentUser().getShortUserName()); + QueueEntity queue = new QueueEntity(); + queue.setId("default_queue"); + client.putEntities(cluster, flow, app, appAttempt, container, user, queue); + client.putEntitiesAsync(cluster, flow, app, appAttempt, container, user, queue); + } finally { + client.stop(); + } + } + private static class MyTimelineCollectorManager extends TimelineCollectorManager { public MyTimelineCollectorManager() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java index edec0d3..700eac4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java @@ -42,7 +42,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; +import org.apache.hadoop.yarn.api.records.timelineservice.*; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -138,7 +138,8 @@ public Response putEntities( LOG.error("Application: "+ appId + " is not found"); throw new NotFoundException(); // different exception? } - collector.putEntities(entities, callerUgi); + + collector.putEntities(processTimelineEntities(entities), callerUgi); return Response.ok().build(); } catch (Exception e) { LOG.error("Error putting entities", e); @@ -147,7 +148,7 @@ public Response putEntities( } } - private String parseApplicationId(String appId) { + private static String parseApplicationId(String appId) { try { if (appId != null) { return ConverterUtils.toApplicationId(appId.trim()).toString(); @@ -168,11 +169,11 @@ private String parseApplicationId(String appId) { return collectorManager.get(appIdString); } - private void init(HttpServletResponse response) { + private static void init(HttpServletResponse response) { response.setContentType(null); } - private UserGroupInformation getUser(HttpServletRequest req) { + private static UserGroupInformation getUser(HttpServletRequest req) { String remoteUser = req.getRemoteUser(); UserGroupInformation callerUgi = null; if (remoteUser != null) { @@ -180,4 +181,46 @@ private UserGroupInformation getUser(HttpServletRequest req) { } return callerUgi; } + + // Depend on how the server + private static TimelineEntities processTimelineEntities( + TimelineEntities entities) { + TimelineEntities entitiesToReturn = new TimelineEntities(); + for (TimelineEntity entity : entities.getEntities()) { + TimelineEntityType type = null; + try { + type = TimelineEntityType.valueOf(entity.getType()); + } catch (IllegalArgumentException e) { + type = null; + } + if (type != null) { + switch (type) { + case YARN_CLUSTER: + entitiesToReturn.addEntity(new ClusterEntity(entity)); + break; + case YARN_FLOW: + entitiesToReturn.addEntity(new FlowEntity(entity)); + break; + case YARN_APPLICATION: + entitiesToReturn.addEntity(new ApplicationEntity(entity)); + break; + case YARN_APPLICATION_ATTEMPT: + entitiesToReturn.addEntity(new ApplicationAttemptEntity(entity)); + break; + case YARN_CONTAINER: + entitiesToReturn.addEntity(new ContainerEntity(entity)); + break; + case YARN_QUEUE: + entitiesToReturn.addEntity(new QueueEntity(entity)); + break; + case YARN_USER: + entitiesToReturn.addEntity(new UserEntity(entity)); + break; + default: + break; + } + } + } + return entitiesToReturn; + } }