From cd4b8f271029ffc2d3f4c4c42200f0d6db7d8433 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 26 Mar 2019 20:46:01 +0530 Subject: [PATCH] YARN-9403 --- .../TestTimelineReaderWebServicesHBaseStorage.java | 16 +++ .../reader/TimelineEntityReaderFactory.java | 52 ++++----- .../reader/TimelineReaderContext.java | 23 +++- .../reader/TimelineReaderWebServices.java | 120 +++++++++++++-------- 4 files changed, 142 insertions(+), 69 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java index abd5362..d885ffc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java @@ -2598,4 +2598,20 @@ public void testGetAppsMetricsRange() throws Exception { client.destroy(); } } + + @Test + public void testGetEntityWithSystemEntityType() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/apps/application_1111111111_1111/" + + "entities/YARN_APPLICATION"); + ClientResponse resp = getResponse(client, uri); + Set entities = + resp.getEntity(new GenericType>(){}); + assertEquals(0, entities.size()); + } finally { + client.destroy(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java index fa16077..77ac0ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java @@ -43,18 +43,20 @@ public static TimelineEntityReader createSingleEntityReader( TimelineReaderContext context, TimelineDataToRetrieve dataToRetrieve) { // currently the types that are handled separate from the generic entity // table are application, flow run, and flow activity entities - if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) { - return new ApplicationEntityReader(context, dataToRetrieve); - } else if (TimelineEntityType. - YARN_FLOW_RUN.matches(context.getEntityType())) { - return new FlowRunEntityReader(context, dataToRetrieve); - } else if (TimelineEntityType. - YARN_FLOW_ACTIVITY.matches(context.getEntityType())) { - return new FlowActivityEntityReader(context, dataToRetrieve); - } else { - // assume we're dealing with a generic entity read - return new GenericEntityReader(context, dataToRetrieve); + if (!context.isGenericEntity()) { + if (TimelineEntityType. + YARN_APPLICATION.matches(context.getEntityType())) { + return new ApplicationEntityReader(context, dataToRetrieve); + } else if (TimelineEntityType. + YARN_FLOW_RUN.matches(context.getEntityType())) { + return new FlowRunEntityReader(context, dataToRetrieve); + } else if (TimelineEntityType. + YARN_FLOW_ACTIVITY.matches(context.getEntityType())) { + return new FlowActivityEntityReader(context, dataToRetrieve); + } } + // assume we're dealing with a generic entity read + return new GenericEntityReader(context, dataToRetrieve); } /** @@ -73,21 +75,23 @@ public static TimelineEntityReader createMultipleEntitiesReader( TimelineDataToRetrieve dataToRetrieve) { // currently the types that are handled separate from the generic entity // table are application, flow run, and flow activity entities - if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) { - return new ApplicationEntityReader(context, filters, dataToRetrieve); - } else if (TimelineEntityType. - YARN_FLOW_ACTIVITY.matches(context.getEntityType())) { - return new FlowActivityEntityReader(context, filters, dataToRetrieve); - } else if (TimelineEntityType. - YARN_FLOW_RUN.matches(context.getEntityType())) { - return new FlowRunEntityReader(context, filters, dataToRetrieve); - } else { - if (context.getDoAsUser() != null) { - return new SubApplicationEntityReader(context, filters, dataToRetrieve); + if (!context.isGenericEntity()) { + if (TimelineEntityType. + YARN_APPLICATION.matches(context.getEntityType())) { + return new ApplicationEntityReader(context, filters, dataToRetrieve); + } else if (TimelineEntityType. + YARN_FLOW_ACTIVITY.matches(context.getEntityType())) { + return new FlowActivityEntityReader(context, filters, dataToRetrieve); + } else if (TimelineEntityType. + YARN_FLOW_RUN.matches(context.getEntityType())) { + return new FlowRunEntityReader(context, filters, dataToRetrieve); } - // assume we're dealing with a generic entity read - return new GenericEntityReader(context, filters, dataToRetrieve); } + if (context.getDoAsUser() != null) { + return new SubApplicationEntityReader(context, filters, dataToRetrieve); + } + // assume we're dealing with a generic entity read + return new GenericEntityReader(context, filters, dataToRetrieve); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java index 67c3d29..62e39d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java @@ -33,6 +33,8 @@ private String entityId; private Long entityIdPrefix; private String doAsUser; + private boolean genericEntity = false; + public TimelineReaderContext(String clusterId, String userId, String flowName, Long flowRunId, String appId, String entityType, String entityId) { super(clusterId, userId, flowName, flowRunId, appId); @@ -55,10 +57,19 @@ public TimelineReaderContext(String clusterId, String userId, String flowName, this.doAsUser = doasUser; } + public TimelineReaderContext(String clusterId, String userId, String flowName, + Long flowRunId, String appId, String entityType, Long entityIdPrefix, + String entityId, String doasUser, boolean genericEntity) { + this(clusterId, userId, flowName, flowRunId, appId, entityType, + entityIdPrefix, entityId, doasUser); + this.genericEntity = genericEntity; + } + public TimelineReaderContext(TimelineReaderContext other) { this(other.getClusterId(), other.getUserId(), other.getFlowName(), other.getFlowRunId(), other.getAppId(), other.getEntityType(), - other.getEntityIdPrefix(), other.getEntityId(), other.getDoAsUser()); + other.getEntityIdPrefix(), other.getEntityId(), other.getDoAsUser(), + other.genericEntity); } @Override @@ -130,4 +141,14 @@ public String getDoAsUser() { public void setDoAsUser(String doAsUser) { this.doAsUser = doAsUser; } + + public boolean isGenericEntity() { + return genericEntity; + } + + public void setGenericEntity(boolean genericEntity) { + this.genericEntity = genericEntity; + } + + } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index 330e1f4..16cdfad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -338,6 +338,7 @@ public TimelineAbout about( } context.setEntityType( TimelineReaderWebServicesUtils.parseStr(entityType)); + context.setGenericEntity(true); entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, @@ -483,7 +484,7 @@ public TimelineAbout about( flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, true); } /** @@ -603,6 +604,23 @@ public TimelineAbout about( @QueryParam("metricstimestart") String metricsTimeStart, @QueryParam("metricstimeend") String metricsTimeEnd, @QueryParam("fromid") String fromId) { + return getEntities(req, res, null, appId, entityType, userId, flowName, + flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, + isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId, true); + } + + public Set getEntities(HttpServletRequest req, + HttpServletResponse res, String clusterId, String appId, + String entityType, String userId, String flowName, + String flowRunId, String limit, String createdTimeStart, + String createdTimeEnd, String relatesTo, String isRelatedTo, + String infofilters, String conffilters, String metricfilters, + String eventfilters, String confsToRetrieve, String metricsToRetrieve, + String fields, String metricsLimit, String metricsTimeStart, + String metricsTimeEnd, String fromId, + boolean genericEntity) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -619,6 +637,7 @@ public TimelineAbout about( TimelineReaderContext context = TimelineReaderWebServicesUtils .createTimelineReaderContext(clusterId, userId, flowName, flowRunId, appId, entityType, null, null); + context.setGenericEntity(genericEntity); entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils .createTimelineEntityFilters(limit, createdTimeStart, @@ -744,6 +763,54 @@ public TimelineEntity getEntity( return entity; } + public TimelineEntity getEntity(HttpServletRequest req, + HttpServletResponse res, String clusterId, String appId, + String entityType, String entityId, String userId, String flowName, + String flowRunId, String confsToRetrieve, String metricsToRetrieve, + String fields, String metricsLimit, String metricsTimeStart, + String metricsTimeEnd, String entityIdPrefix, + boolean genericEntity) { + String url = req.getRequestURI() + + (req.getQueryString() == null ? "" : + QUERY_STRING_SEP + req.getQueryString()); + UserGroupInformation callerUGI = + TimelineReaderWebServicesUtils.getUser(req); + LOG.info("Received URL " + url + " from user " + + TimelineReaderWebServicesUtils.getUserName(callerUGI)); + long startTime = Time.monotonicNow(); + boolean succeeded = false; + init(res); + TimelineReaderManager timelineReaderManager = getTimelineReaderManager(); + TimelineEntity entity = null; + TimelineReaderContext context = TimelineReaderWebServicesUtils. + createTimelineReaderContext(clusterId, userId, flowName, flowRunId, + appId, entityType, entityIdPrefix, entityId); + context.setGenericEntity(genericEntity); + try { + entity = timelineReaderManager.getEntity(context, + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); + checkAccessForGenericEntity(entity, callerUGI); + succeeded = true; + } catch (Exception e) { + handleException(e, url, startTime, "Either flowrunid or metricslimit or" + + " metricstime start/end"); + } finally { + long latency = Time.monotonicNow() - startTime; + METRICS.addGetEntitiesLatency(latency, succeeded); + LOG.info("Processed URL " + url + + " (Took " + latency + " ms.)"); + } + if (entity == null) { + LOG.info("Processed URL " + url + " but entity not found" + " (Took " + + (Time.monotonicNow() - startTime) + " ms.)"); + throw new NotFoundException("Timeline entity {id: " + entityId + + ", type: " + entityType + " } is not found"); + } + return entity; + } + /** * Return a single entity of the given entity type and Id. Cluster ID is not * provided by client so default cluster ID has to be taken. If userid, flow @@ -820,7 +887,7 @@ public TimelineEntity getEntity( @QueryParam("entityidprefix") String entityIdPrefix) { return getEntity(req, res, null, appId, entityType, entityId, userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, - metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix); + metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, true); } /** @@ -899,44 +966,9 @@ public TimelineEntity getEntity( @QueryParam("metricstimestart") String metricsTimeStart, @QueryParam("metricstimeend") String metricsTimeEnd, @QueryParam("entityidprefix") String entityIdPrefix) { - String url = req.getRequestURI() + - (req.getQueryString() == null ? "" : - QUERY_STRING_SEP + req.getQueryString()); - UserGroupInformation callerUGI = - TimelineReaderWebServicesUtils.getUser(req); - LOG.info("Received URL " + url + " from user " + - TimelineReaderWebServicesUtils.getUserName(callerUGI)); - long startTime = Time.monotonicNow(); - boolean succeeded = false; - init(res); - TimelineReaderManager timelineReaderManager = getTimelineReaderManager(); - TimelineEntity entity = null; - try { - entity = timelineReaderManager.getEntity( - TimelineReaderWebServicesUtils.createTimelineReaderContext( - clusterId, userId, flowName, flowRunId, appId, entityType, - entityIdPrefix, entityId), - TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd)); - checkAccessForGenericEntity(entity, callerUGI); - succeeded = true; - } catch (Exception e) { - handleException(e, url, startTime, "Either flowrunid or metricslimit or" - + " metricstime start/end"); - } finally { - long latency = Time.monotonicNow() - startTime; - METRICS.addGetEntitiesLatency(latency, succeeded); - LOG.info("Processed URL " + url + - " (Took " + latency + " ms.)"); - } - if (entity == null) { - LOG.info("Processed URL " + url + " but entity not found" + " (Took " + - (Time.monotonicNow() - startTime) + " ms.)"); - throw new NotFoundException("Timeline entity {id: " + entityId + - ", type: " + entityType + " } is not found"); - } - return entity; + return getEntity(req, res, clusterId, appId, entityType, entityId, + userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, + metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, true); } /** @@ -2055,7 +2087,7 @@ public TimelineEntity getApp( flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, false); } /** @@ -2169,7 +2201,7 @@ public TimelineEntity getApp( flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, false); } /** @@ -2277,7 +2309,7 @@ public TimelineEntity getApp( null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, false); } /** @@ -2387,7 +2419,7 @@ public TimelineEntity getApp( null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, false); } /** -- 2.7.4 (Apple Git-66)