diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java index 0938e9e..b56a752 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type; +import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; final class DataGeneratorForTest { static void loadApps(HBaseTestingUtility util) throws IOException { @@ -358,6 +359,46 @@ static void loadEntities(HBaseTestingUtility util) throws IOException { relatesTo3.put("container2", relatesToSet14); entity2.setRelatesToEntities(relatesTo3); te.addEntity(entity2); + + // For listing types + for (int i = 0; i < 10; i++) { + TimelineEntity entity3 = new TimelineEntity(); + String id3 = "typeTest" + i; + entity3.setId(id3); + StringBuilder typeName = new StringBuilder("newType"); + for (int j = 0; j < (i % 3); j++) { + typeName.append(" ").append(j); + } + entity3.setType(typeName.toString()); + entity3.setCreatedTime(cTime + 80L + i); + te.addEntity(entity3); + } + + // Create app entity for app to flow table + TimelineEntities appTe1 = new TimelineEntities(); + TimelineEntity entityApp1 = new TimelineEntity(); + String appName1 = "application_1231111111_1111"; + entityApp1.setId(appName1); + entityApp1.setType(TimelineEntityType.YARN_APPLICATION.toString()); + entityApp1.setCreatedTime(cTime + 40L); + TimelineEvent appCreationEvent1 = new TimelineEvent(); + appCreationEvent1.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE); + appCreationEvent1.setTimestamp(cTime); + entityApp1.addEvent(appCreationEvent1); + appTe1.addEntity(entityApp1); + + TimelineEntities appTe2 = new TimelineEntities(); + TimelineEntity entityApp2 = new TimelineEntity(); + String appName2 = "application_1231111111_1112"; + entityApp2.setId(appName2); + entityApp2.setType(TimelineEntityType.YARN_APPLICATION.toString()); + entityApp2.setCreatedTime(cTime + 50L); + TimelineEvent appCreationEvent2 = new TimelineEvent(); + appCreationEvent2.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE); + appCreationEvent2.setTimestamp(cTime); + entityApp2.addEvent(appCreationEvent2); + appTe2.addEntity(entityApp2); + HBaseTimelineWriterImpl hbi = null; try { hbi = new HBaseTimelineWriterImpl(util.getConfiguration()); @@ -368,8 +409,10 @@ static void loadEntities(HBaseTestingUtility util) throws IOException { String flow = "some_flow_name"; String flowVersion = "AB7822C10F1111"; long runid = 1002345678919L; - String appName = "application_1231111111_1111"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(cluster, user, flow, flowVersion, runid, appName1, te); + hbi.write(cluster, user, flow, flowVersion, runid, appName2, te); + hbi.write(cluster, user, flow, flowVersion, runid, appName1, appTe1); + hbi.write(cluster, user, flow, flowVersion, runid, appName2, appTe2); hbi.stop(); } finally { if (hbi != null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java index 3076709..4316a50 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java @@ -1668,6 +1668,29 @@ public void testReadEntitiesInfoFilters() throws Exception { assertEquals(3, entities.size()); } + @Test(timeout = 90000) + public void testListAppsInApp() throws Exception { + Set types = reader.listEntityTypes( + new TimelineReaderContext("cluster1", "user1", "some_flow_name", + 1002345678919L, "application_1231111111_1111", null, null)); + assertEquals(4, types.size()); + + types = reader.listEntityTypes( + new TimelineReaderContext("cluster1", null, null, + null, "application_1231111111_1111", null, null)); + assertEquals(4, types.size()); + + types = reader.listEntityTypes( + new TimelineReaderContext("cluster1", null, null, + null, "application_1231111111_1112", null, null)); + assertEquals(4, types.size()); + + types = reader.listEntityTypes( + new TimelineReaderContext("cluster1", "user1", "some_flow_name", + 1002345678919L, "application_1231111111_1113", null, null)); + assertEquals(0, types.size()); + } + @AfterClass public static void tearDownAfterClass() throws Exception { util.shutdownMiniCluster(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java index 4cff3bc..63451f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.timelineservice.reader; import java.io.IOException; +import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -176,4 +178,25 @@ public TimelineEntity getEntity(TimelineReaderContext context, } return entity; } + + /** + * Get a list for available timeline entity types by making a call to backend + * storage implementation. The meaning of each argument in detail is the same + * as {@link TimelineReader#getEntity}. If cluster ID has not + * been supplied by the client, fills the cluster id from config before making + * a call to backend storage. + * + * @param context Timeline context within the scope of which entity types + * have to be fetched. Entity type field of this context should + * be null. + * @return A set contains available timeine entity types, represented as + * Strings if found, null otherwise. + * @throws IOException if any problem occurs while getting entity. + */ + public Set listEntityTypes(TimelineReaderContext context) + throws IOException{ + context.setClusterId(getClusterID(context.getClusterId(), getConfig())); + return reader.listEntityTypes( + new TimelineReaderContext(context)); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index db0c4e1..7b8ca48 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -2859,4 +2859,111 @@ public TimelineEntity getContainer(@Context HttpServletRequest req, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, metricsLimit); } + + /** + * Return a set of available entity types for a given app id. Cluster ID is + * not provided by client so default cluster ID has to be taken. If userid, + * flow name and flow run id which are optional query parameters are not + * specified, they will be queried based on app id and cluster id from the + * flow context information stored in underlying storage implementation. + * + * @param req Servlet request. + * @param res Servlet response. + * @param appId Application id to be queried(Mandatory path param). + * @param flowName Flow name which should match for the app(Optional query + * param). + * @param flowRunId Run id which should match for the app(Optional query + * param). + * @param userId User id which should match for the app(Optional query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * list contains all timeline entity types is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved or app for the given + * app id cannot be found, HTTP 404(Not Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server + * Error) is returned. + */ + @GET + @Path("/apps/{appid}/entities") + @Produces(MediaType.APPLICATION_JSON) + public Set getEntityTypes( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("appid") String appId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("userid") String userId) { + return getEntityTypes(req, res, null, appId, flowName, flowRunId, userId); + } + + /** + * Return a set of available entity types for a given app id. If userid, + * flow name and flow run id which are optional query parameters are not + * specified, they will be queried based on app id and cluster id from the + * flow context information stored in underlying storage implementation. + * + * @param req Servlet request. + * @param res Servlet response. + * @param clusterId Cluster id to which the app to be queried belong to( + * Mandatory path param). + * @param appId Application id to be queried(Mandatory path param). + * @param flowName Flow name which should match for the app(Optional query + * param). + * @param flowRunId Run id which should match for the app(Optional query + * param). + * @param userId User id which should match for the app(Optional query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * list contains all timeline entity types is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved or app for the given + * app id cannot be found, HTTP 404(Not Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server + * Error) is returned. + */ + @GET + @Path("/clusters/{clusterid}/apps/{appid}/entities") + @Produces(MediaType.APPLICATION_JSON) + public Set getEntityTypes( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("clusterid") String clusterId, + @PathParam("appid") String appId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("userid") String userId) { + String url = req.getRequestURI() + + (req.getQueryString() == null ? "" : + QUERY_STRING_SEP + req.getQueryString()); + UserGroupInformation callerUGI = + TimelineReaderWebServicesUtils.getUser(req); + LOG.info("Received URL " + url + " from user " + + TimelineReaderWebServicesUtils.getUserName(callerUGI)); + long startTime = Time.monotonicNow(); + init(res); + TimelineReaderManager timelineReaderManager = getTimelineReaderManager(); + Set results = null; + try { + results = timelineReaderManager.listEntityTypes( + TimelineReaderWebServicesUtils.createTimelineReaderContext( + clusterId, userId, flowName, flowRunId, appId, + null, null)); + } catch (Exception e) { + handleException(e, url, startTime, "flowrunid"); + } + long endTime = Time.monotonicNow(); + if (results == null) { + LOG.info("Processed URL " + url + " but app not found" + " (Took " + + (endTime - startTime) + " ms.)"); + throw new NotFoundException("App " + appId + " not found"); + } + LOG.info("Processed URL " + url + + " (Took " + (endTime - startTime) + " ms.)"); + return results; + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index ebb73b3..90a72c9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -32,6 +32,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -403,4 +404,24 @@ public TimelineEntity getEntity(TimelineReaderContext context, context.getEntityType()); return getEntities(dir, context.getEntityType(), filters, dataToRetrieve); } + + @Override public Set listEntityTypes(TimelineReaderContext context) + throws IOException { + Set result = new TreeSet<>(); + String flowRunPath = getFlowRunPath(context.getUserId(), + context.getClusterId(), context.getFlowName(), context.getFlowRunId(), + context.getAppId()); + File dir = new File(new File(rootPath, ENTITIES_DIR), + context.getClusterId() + File.separator + flowRunPath + + File.separator + context.getAppId()); + File[] fileList = dir.listFiles(); + if (fileList != null) { + for (File f : fileList) { + if (f.isDirectory()) { + result.add(f.getName()); + } + } + } + return result; + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java index a384a84..9b41a54 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java @@ -19,7 +19,9 @@ import java.io.IOException; +import java.util.HashSet; import java.util.Set; +import java.util.TreeSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -85,4 +87,20 @@ public TimelineEntity getEntity(TimelineReaderContext context, filters, dataToRetrieve); return reader.readEntities(hbaseConf, conn); } + + @Override + public Set listEntityTypes(TimelineReaderContext context) + throws IOException { + TimelineEntityReader reader + = TimelineEntityReaderFactory.createEntityTypeReader(context); + Set entities = reader.readEntities(hbaseConf, conn); + Set types = new TreeSet<>(); + for (TimelineEntity e : entities) { + if (!types.add(e.getType())) { + LOG.warn("Failed to add type " + e.getType() + " to the result set " + + "because there is a duplicated copy. "); + } + } + return types; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java index e8eabf1..b67f66e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java @@ -177,4 +177,17 @@ TimelineEntity getEntity(TimelineReaderContext context, TimelineReaderContext context, TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve) throws IOException; + + /** + * The API to list all available entity types of the given context. + * + * @param context A context defines the scope of this query. The incoming + * context should contain at least the cluster id and application id. + * + * @return A set of entity types available in the given context. + * + * @throws IOException if an exception occurred while listing from backend + * storage. + */ + Set listEntityTypes(TimelineReaderContext context) throws IOException; } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java new file mode 100644 index 0000000..c854627 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.reader; + +import com.google.common.base.Preconditions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; +import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * Timeline entity reader for listing all available entity types given one + * reader context. Right now only supports listing all entity types within one + * YARN application. + */ +final class EntityTypeReader extends GenericEntityReader { + + private static final Log LOG = LogFactory.getLog(EntityTypeReader.class); + public EntityTypeReader(TimelineReaderContext context) { + super(context, new TimelineDataToRetrieve()); + } + + /** + * Reads a set of timeline entities with different types from the HBase + * storage. + * + * @param hbaseConf HBase Configuration. + * @param conn HBase Connection. + * @return a set of TimelineEntity objects, with only type field + * set. + * @throws IOException if any exception is encountered while reading entities. + */ + @Override + public Set readEntities(Configuration hbaseConf, + Connection conn) throws IOException { + + validateParams(); + augmentParams(hbaseConf, conn); + + // Use HashSet since the entity we add here may miss some data for ordering. + Set entities = new HashSet<>(); + TimelineReaderContext context = getContext(); + EntityRowKeyPrefix prefix = new EntityRowKeyPrefix(context.getClusterId(), + context.getUserId(), context.getFlowName(), context.getFlowRunId(), + context.getAppId()); + byte[] currRowKey = prefix.getRowKeyPrefix(); + + FilterList typeFilterList = new FilterList(); + typeFilterList.addFilter(new FirstKeyOnlyFilter()); + typeFilterList.addFilter(new PrefixFilter(currRowKey)); + if (LOG.isDebugEnabled()) { + LOG.debug("FilterList created for scan is - " + typeFilterList); + } + + int counter = 0; + while (true) { + try (ResultScanner results + = getResult(hbaseConf, conn, typeFilterList, currRowKey)) { + TimelineEntity entity = parseEntityForType(results.next()); + if (entity == null) { + break; + } + ++counter; + entities.add(entity); + String currType = entity.getType(); + if (LOG.isDebugEnabled()) { + LOG.debug("Current row key: " + Arrays.toString(currRowKey)); + LOG.debug("New entity type discovered: " + currType); + } + currRowKey = getNextRowKey(prefix.getRowKeyPrefix(), currType); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Scanned " + counter + "records for " + + entities.size() + "types"); + } + return entities; + } + + @Override + protected void validateParams() { + Preconditions.checkNotNull(getContext(), "context shouldn't be null"); + Preconditions.checkNotNull(getContext().getClusterId(), + "clusterId shouldn't be null"); + Preconditions.checkNotNull(getContext().getAppId(), + "appId shouldn't be null"); + } + + /** + * Get the possibly next row key prefix given current prefix and type. + * + * @param currRowKeyPrefix The current prefix that contains user, cluster, + * flow, run, and application information. Should not + * be changed during the iteration. + * @param entityType Current entity type. + * @return A new prefix for the possibly immediately next row key. + */ + private static byte[] getNextRowKey(byte[] currRowKeyPrefix, + String entityType) { + if (currRowKeyPrefix == null || entityType == null) { + return null; + } + + // The last byte represents the separator. The immediate next possible row + // key should have a byte with at least qualifier + 1. + // (Otherwise it will be stored before the current type. ) + byte[] nextTypeEncoded = Separator.QUALIFIERS.join( + Separator.encode(entityType, Separator.SPACE, Separator.TAB, + Separator.QUALIFIERS), + Separator.EMPTY_BYTES); + nextTypeEncoded[nextTypeEncoded.length - 1] += 1; + + byte[] nextRowKey + = new byte[currRowKeyPrefix.length + nextTypeEncoded.length]; + System.arraycopy(currRowKeyPrefix, 0, nextRowKey, 0, + currRowKeyPrefix.length); + System.arraycopy(nextTypeEncoded, 0, nextRowKey, currRowKeyPrefix.length, + nextTypeEncoded.length); + + return nextRowKey; + } + + private ResultScanner getResult(Configuration hbaseConf, Connection conn, + FilterList filterList, byte[] startPrefix) + throws IOException { + Scan scan = new Scan(startPrefix); + scan.setMaxVersions(getDataToRetrieve().getMetricsLimit()); + scan.setFilter(filterList); + scan.setSmall(true); + return getTable().getResultScanner(hbaseConf, conn, scan); + } + + private TimelineEntity parseEntityForType(Result result) + throws IOException { + if (result == null || result.isEmpty()) { + return null; + } + TimelineEntity entity = new TimelineEntity(); + EntityRowKey newRowKey = EntityRowKey.parseRowKey(result.getRow()); + entity.setType(newRowKey.getEntityType()); + return entity; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java index b2a9476..e90338e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java @@ -86,4 +86,17 @@ public static TimelineEntityReader createMultipleEntitiesReader( return new GenericEntityReader(context, filters, dataToRetrieve, false); } } + + /** + * Creates a timeline entity type reader that will read all available entity + * types within the specified context. + * + * @param context Reader context which defines the scope in which query has to + * be made. Limited to application level only. + * @return an EntityTypeReader object + */ + public static EntityTypeReader createEntityTypeReader( + TimelineReaderContext context) { + return new EntityTypeReader(context); + } }