From 251009c55ab4f2d9209dfed898fe828cc682d641 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 26 Mar 2019 17:17:24 +0530 Subject: [PATCH] YARN-9403 --- .../TestTimelineReaderWebServicesHBaseStorage.java | 17 +++ .../storage/TestHBaseTimelineStorageApps.java | 154 ++++++++++++--------- .../storage/TestHBaseTimelineStorageEntities.java | 4 +- .../storage/TestTimelineReaderHBaseDown.java | 3 +- .../storage/flow/TestHBaseStorageFlowActivity.java | 9 +- .../storage/flow/TestHBaseStorageFlowRun.java | 43 ++++-- .../storage/HBaseTimelineReaderImpl.java | 3 +- .../reader/TimelineEntityReaderFactory.java | 12 +- .../reader/TimelineReaderContext.java | 35 ++++- .../reader/TimelineReaderWebServices.java | 91 +++++++++--- .../reader/TimelineReaderWebServicesUtils.java | 26 ++-- 11 files changed, 267 insertions(+), 130 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java index abd5362..c580b3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java @@ -2598,4 +2598,21 @@ public void testGetAppsMetricsRange() throws Exception { client.destroy(); } } + + @Test + public void testGetEntityWithSystemEntityType() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/apps/application_1111111111_1111/" + + "entities/YARN_APPLICATION"); + ClientResponse resp = getResponse(client, uri); + Set entities = + resp.getEntity(new GenericType>(){}); + assertEquals(0, entities.size()); + } finally { + client.destroy(); + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java index 0dee442..c1ae704 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java @@ -383,7 +383,8 @@ public void testWriteApplicationToHBase() throws Exception { // one value exists for a metric). TimelineEntity e1 = reader.getEntity( new TimelineReaderContext(cluster, user, flow, runid, appId, - entity.getType(), entity.getId()), + entity.getType(), null, entity.getId(), null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve(null, null, EnumSet.of(TimelineReader.Field.ALL), Integer.MAX_VALUE, null, null)); assertNotNull(e1); @@ -426,7 +427,8 @@ public void testWriteApplicationToHBase() throws Exception { // In metrics limit specify a value of 3. No more than 3 values for a // metric will be returned. e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, - runid, appId, entity.getType(), entity.getId()), + runid, appId, entity.getType(), null, entity.getId(), null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve(null, null, EnumSet.of(TimelineReader.Field.ALL), 3, null, null)); assertNotNull(e1); @@ -448,7 +450,8 @@ public void testWriteApplicationToHBase() throws Exception { // values. e1 = reader.getEntity( new TimelineReaderContext(cluster, user, flow, runid, appId, - entity.getType(), entity.getId()), new TimelineDataToRetrieve( + entity.getType(), null, entity.getId(), null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve( null, null, EnumSet.of(TimelineReader.Field.ALL), null, null, null)); assertNotNull(e1); assertEquals(appId, e1.getId()); @@ -560,12 +563,14 @@ public void testEvents() throws IOException { // read the timeline entity using the reader this time TimelineEntity e1 = reader.getEntity( new TimelineReaderContext(cluster, user, flow, runid, appName, - entity.getType(), entity.getId()), + entity.getType(), null, entity.getId(), null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, null, null)); TimelineEntity e2 = reader.getEntity( new TimelineReaderContext(cluster, user, null, null, appName, - entity.getType(), entity.getId()), + entity.getType(), null, entity.getId(), null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, null, null)); assertNotNull(e1); @@ -663,7 +668,8 @@ public void testReadApps() throws Exception { TimelineEntity entity = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", - TimelineEntityType.YARN_APPLICATION.toString(), null), + TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, null, null)); assertNotNull(entity); @@ -672,7 +678,7 @@ public void testReadApps() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, null, null)); @@ -710,7 +716,7 @@ public void testFilterAppsByCreatedTime() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().createdTimeBegin(1425016502000L) .createTimeEnd(1425016502040L).build(), new TimelineDataToRetrieve()); @@ -727,7 +733,7 @@ public void testFilterAppsByCreatedTime() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().createdTimeBegin(1425016502015L) .build(), new TimelineDataToRetrieve()); @@ -742,7 +748,7 @@ public void testFilterAppsByCreatedTime() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().createTimeEnd(1425016502015L) .build(), new TimelineDataToRetrieve()); @@ -760,7 +766,8 @@ public void testReadAppsDefaultView() throws Exception { TimelineEntity e1 = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", - TimelineEntityType.YARN_APPLICATION.toString(), null), + TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve()); assertNotNull(e1); assertEquals(1, e1.getInfo().size()); @@ -770,7 +777,7 @@ public void testReadAppsDefaultView() throws Exception { Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve()); assertEquals(3, es1.size()); @@ -787,7 +794,8 @@ public void testReadAppsByFields() throws Exception { TimelineEntity e1 = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", - TimelineEntityType.YARN_APPLICATION.toString(), null), + TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve( null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null, null, null)); assertNotNull(e1); @@ -796,7 +804,8 @@ public void testReadAppsByFields() throws Exception { Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), new TimelineEntityFilters.Builder().build(), + null, null, null, TimelineEntityType.YARN_APPLICATION), + new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve( null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS), null, null, null)); @@ -826,7 +835,7 @@ public void testReadAppsIsRelatedTo() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().isRelatedTo(irt).build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, null, null)); @@ -852,7 +861,7 @@ public void testReadAppsIsRelatedTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().isRelatedTo(irt1).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -875,7 +884,7 @@ public void testReadAppsIsRelatedTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().isRelatedTo(irt2).build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); @@ -897,7 +906,7 @@ public void testReadAppsIsRelatedTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().isRelatedTo(irt3).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -920,7 +929,7 @@ public void testReadAppsIsRelatedTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().isRelatedTo(irt4).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -932,7 +941,7 @@ public void testReadAppsIsRelatedTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().isRelatedTo(irt5).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -952,7 +961,7 @@ public void testReadAppsIsRelatedTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().isRelatedTo(irt6).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -979,7 +988,7 @@ public void testReadAppsRelatesTo() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().relatesTo(rt).build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, null, null)); @@ -1005,7 +1014,7 @@ public void testReadAppsRelatesTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().relatesTo(rt1).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -1028,7 +1037,7 @@ public void testReadAppsRelatesTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().relatesTo(rt2).build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); @@ -1050,7 +1059,7 @@ public void testReadAppsRelatesTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().relatesTo(rt3).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -1073,7 +1082,7 @@ public void testReadAppsRelatesTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().relatesTo(rt4).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -1085,7 +1094,7 @@ public void testReadAppsRelatesTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().relatesTo(rt5).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -1105,7 +1114,7 @@ public void testReadAppsRelatesTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().relatesTo(rt6).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -1141,7 +1150,7 @@ public void testReadAppsRelatesTo() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().relatesTo(rt7).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -1178,7 +1187,7 @@ public void testReadAppsRelationsAndEventFiltersDefaultView() Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().relatesTo(relatesTo) .isRelatedTo(isRelatedTo).eventFilters(eventFilter).build(), new TimelineDataToRetrieve()); @@ -1216,7 +1225,7 @@ public void testReadAppsConfigFilters() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().configFilters(confFilterList) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), @@ -1231,7 +1240,7 @@ public void testReadAppsConfigFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().configFilters(confFilterList) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, @@ -1249,7 +1258,7 @@ public void testReadAppsConfigFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().configFilters(confFilterList1) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), @@ -1269,7 +1278,7 @@ public void testReadAppsConfigFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().configFilters(confFilterList2) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), @@ -1282,7 +1291,7 @@ public void testReadAppsConfigFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().configFilters(confFilterList3) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), @@ -1295,7 +1304,7 @@ public void testReadAppsConfigFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().configFilters(confFilterList4) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), @@ -1308,7 +1317,7 @@ public void testReadAppsConfigFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().configFilters(confFilterList5) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), @@ -1326,7 +1335,7 @@ public void testReadAppsEventFilters() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().eventFilters(ef).build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, null, null)); @@ -1348,7 +1357,7 @@ public void testReadAppsEventFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().eventFilters(ef1).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -1367,7 +1376,7 @@ public void testReadAppsEventFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().eventFilters(ef2).build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); @@ -1390,7 +1399,7 @@ public void testReadAppsEventFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().eventFilters(ef3).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -1407,7 +1416,7 @@ public void testReadAppsEventFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().eventFilters(ef4).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -1428,7 +1437,7 @@ public void testReadAppsEventFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().eventFilters(ef5).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -1450,14 +1459,15 @@ public void testReadAppsConfigPrefix() throws Exception { TimelineEntity e1 = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", - TimelineEntityType.YARN_APPLICATION.toString(), null), + TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve(list, null, null, null, null, null)); assertNotNull(e1); assertEquals(1, e1.getConfigs().size()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null) , + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve(list, null, null, null, null, null)); int cfgCnt = 0; @@ -1482,7 +1492,7 @@ public void testReadAppsConfigFilterPrefix() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().configFilters(confFilterList) .build(), new TimelineDataToRetrieve(list, null, null, null, null, null)); @@ -1515,7 +1525,7 @@ public void testReadAppsConfigFilterPrefix() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().configFilters(confFilterList1) .build(), new TimelineDataToRetrieve(confsToRetrieve, null, null, null, null, @@ -1547,7 +1557,7 @@ public void testReadAppsMetricFilters() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), @@ -1562,7 +1572,7 @@ public void testReadAppsMetricFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, @@ -1582,7 +1592,7 @@ public void testReadAppsMetricFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), @@ -1602,7 +1612,7 @@ public void testReadAppsMetricFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList2) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), @@ -1615,7 +1625,7 @@ public void testReadAppsMetricFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList3) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), @@ -1628,7 +1638,7 @@ public void testReadAppsMetricFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList4) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), @@ -1641,7 +1651,7 @@ public void testReadAppsMetricFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList5) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), @@ -1657,14 +1667,15 @@ public void testReadAppsMetricPrefix() throws Exception { TimelineEntity e1 = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", - TimelineEntityType.YARN_APPLICATION.toString(), null), + TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve(null, list, null, null, null, null)); assertNotNull(e1); assertEquals(1, e1.getMetrics().size()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve(null, list, null, null, null, null)); int metricCnt = 0; @@ -1689,7 +1700,7 @@ public void testReadAppsMetricFilterPrefix() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList) .build(), new TimelineDataToRetrieve(null, list, null, null, null, null)); @@ -1715,7 +1726,7 @@ public void testReadAppsMetricFilterPrefix() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) .build(), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null, null, @@ -1733,7 +1744,8 @@ public void testReadAppsMetricFilterPrefix() throws Exception { entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, - TimelineEntityType.YARN_APPLICATION.toString(), null), + TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, + TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) .build(), new TimelineDataToRetrieve(null, metricsToRetrieve, @@ -1758,7 +1770,8 @@ public void testReadAppsMetricTimeRange() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), new TimelineEntityFilters.Builder().build(), + null, null, null, TimelineEntityType.YARN_APPLICATION), + new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), 100, null, null)); assertEquals(3, entities.size()); @@ -1776,7 +1789,8 @@ public void testReadAppsMetricTimeRange() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), new TimelineEntityFilters.Builder().build(), + null, null, null, TimelineEntityType.YARN_APPLICATION), + new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), 100, CURRENT_TIME - 40000, CURRENT_TIME)); assertEquals(3, entities.size()); @@ -1797,7 +1811,8 @@ public void testReadAppsMetricTimeRange() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), new TimelineEntityFilters.Builder().build(), + null, null, null, TimelineEntityType.YARN_APPLICATION), + new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), null, CURRENT_TIME - 40000, CURRENT_TIME)); assertEquals(3, entities.size()); @@ -1818,7 +1833,8 @@ public void testReadAppsMetricTimeRange() throws Exception { TimelineEntity entity = reader.getEntity(new TimelineReaderContext( "cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", - TimelineEntityType.YARN_APPLICATION.toString(), null), + TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), 100, CURRENT_TIME - 40000, CURRENT_TIME)); assertNotNull(entity); @@ -1850,7 +1866,7 @@ public void testReadAppsInfoFilters() throws Exception { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().infoFilters(infoFilterList).build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, null, null)); @@ -1867,7 +1883,7 @@ public void testReadAppsInfoFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().infoFilters(infoFilterList1) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, @@ -1887,7 +1903,7 @@ public void testReadAppsInfoFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().infoFilters(infoFilterList2) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, @@ -1900,7 +1916,7 @@ public void testReadAppsInfoFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().infoFilters(infoFilterList3) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, @@ -1913,7 +1929,7 @@ public void testReadAppsInfoFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().infoFilters(infoFilterList4) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, @@ -1926,7 +1942,7 @@ public void testReadAppsInfoFilters() throws Exception { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), + null, null, null, TimelineEntityType.YARN_APPLICATION), new TimelineEntityFilters.Builder().infoFilters(infoFilterList5) .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java index 116285c..130c6c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type; @@ -639,7 +640,8 @@ public void testEventsEscapeTs() throws IOException { // read the timeline entity using the reader this time TimelineEntity e1 = reader.getEntity( new TimelineReaderContext(cluster, user, flow, runid, appName, - entity.getType(), entity.getId()), + entity.getType(), null, entity.getId(), null, + TimelineEntityType.YARN_APPLICATION), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, null, null)); assertNotNull(e1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java index 786f529..084a9f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java @@ -172,7 +172,8 @@ private static void checkQuery(HBaseTimelineReaderImpl htr) throws TimelineReaderContext context = new TimelineReaderContext(YarnConfiguration.DEFAULT_RM_CLUSTER_ID, null, null, null, null, TimelineEntityType - .YARN_FLOW_ACTIVITY.toString(), null, null); + .YARN_FLOW_ACTIVITY.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_ACTIVITY); Set entities = htr.getEntities(context, MONITOR_FILTERS, DATA_TO_RETRIEVE); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java index 3143463..6e1bc85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java @@ -192,7 +192,8 @@ public void testWriteFlowRunMinMax() throws Exception { // get the flow activity entity Set entities = hbr.getEntities( new TimelineReaderContext(cluster, null, null, null, null, - TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), + TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_ACTIVITY), new TimelineEntityFilters.Builder().entityLimit(10L).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -257,7 +258,8 @@ public void testWriteFlowActivityOneFlow() throws Exception { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, - TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), + TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_ACTIVITY), new TimelineEntityFilters.Builder().entityLimit(10L).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -391,7 +393,8 @@ public void testFlowActivityTableOneFlowMultipleRunIds() throws IOException { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, null, null, null, null, - TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), + TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_ACTIVITY), new TimelineEntityFilters.Builder().entityLimit(10L).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java index c3ee758..deabfa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java @@ -247,7 +247,8 @@ public void testWriteFlowRunMinMax() throws Exception { // get the flow run entity TimelineEntity entity = hbr.getEntity( new TimelineReaderContext(cluster, user, flow, runid, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineDataToRetrieve()); assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType())); FlowRunEntity flowRun = (FlowRunEntity)entity; @@ -321,8 +322,8 @@ public void testWriteFlowRunMetricsOneFlow() throws Exception { hbr.start(); TimelineEntity entity = hbr.getEntity( new TimelineReaderContext(cluster, user, flow, runid, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineDataToRetrieve()); + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineDataToRetrieve()); assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType())); Set metrics = entity.getMetrics(); assertEquals(2, metrics.size()); @@ -598,7 +599,8 @@ public void testWriteFlowRunMetricsPrefix() throws Exception { METRIC1.substring(0, METRIC1.indexOf("_") + 1))); TimelineEntity entity = hbr.getEntity( new TimelineReaderContext(cluster, user, flow, 1002345678919L, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null, null, null)); assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType())); @@ -623,7 +625,8 @@ public void testWriteFlowRunMetricsPrefix() throws Exception { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null, null, null)); @@ -690,7 +693,8 @@ public void testWriteFlowRunsMetricFields() throws Exception { hbr.start(); Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, runid, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); @@ -700,7 +704,8 @@ public void testWriteFlowRunsMetricFields() throws Exception { entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, runid, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), null, null, null)); @@ -886,7 +891,8 @@ public void testFilterFlowRunsByCreatedTime() throws Exception { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, - null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), + null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, + null, null, TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().createdTimeBegin(1425016501000L) .createTimeEnd(1425016502001L).build(), new TimelineDataToRetrieve()); @@ -900,7 +906,8 @@ public void testFilterFlowRunsByCreatedTime() throws Exception { } entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().createdTimeBegin(1425016501050L) .build(), new TimelineDataToRetrieve()); @@ -912,7 +919,8 @@ public void testFilterFlowRunsByCreatedTime() throws Exception { } entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().createTimeEnd(1425016501050L) .build(), new TimelineDataToRetrieve()); @@ -987,7 +995,8 @@ public void testMetricFilters() throws Exception { new TimelineFilterList(Operator.OR, list1, list2); Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, - null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), + null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().metricFilters(metricFilterList) .build(), new TimelineDataToRetrieve(null, null, @@ -1005,7 +1014,8 @@ public void testMetricFilters() throws Exception { new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL, METRIC2, 30)); entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) .build(), new TimelineDataToRetrieve(null, null, @@ -1022,7 +1032,8 @@ public void testMetricFilters() throws Exception { new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL, METRIC2, 57)); entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().metricFilters(metricFilterList2) .build(), new TimelineDataToRetrieve(null, null, @@ -1033,7 +1044,8 @@ public void testMetricFilters() throws Exception { new TimelineCompareFilter(TimelineCompareOp.EQUAL, "s_metric", 32)); entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().metricFilters(metricFilterList3) .build(), new TimelineDataToRetrieve(null, null, @@ -1055,7 +1067,8 @@ public void testMetricFilters() throws Exception { METRIC2.substring(0, METRIC2.indexOf("_") + 1))); entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, + TimelineEntityType.YARN_FLOW_RUN), new TimelineEntityFilters.Builder().metricFilters(metricFilterList4) .build(), new TimelineDataToRetrieve(null, metricsToRetrieve, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java index d00ae4b..29eaf23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java @@ -73,7 +73,8 @@ public void serviceInit(Configuration conf) throws Exception { YarnConfiguration.DEFAULT_RM_CLUSTER_ID); monitorContext = new TimelineReaderContext(clusterId, null, null, null, null, - TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null, null); + TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null, null, + null, TimelineEntityType.YARN_FLOW_ACTIVITY); monitorInterval = conf.getLong( YarnConfiguration.TIMELINE_SERVICE_READER_STORAGE_MONITOR_INTERVAL_MS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_STORAGE_MONITOR_INTERVAL_MS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java index fa16077..bb81a5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java @@ -43,13 +43,13 @@ public static TimelineEntityReader createSingleEntityReader( TimelineReaderContext context, TimelineDataToRetrieve dataToRetrieve) { // currently the types that are handled separate from the generic entity // table are application, flow run, and flow activity entities - if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) { + if (TimelineEntityType.YARN_APPLICATION == context.getSystemEntityType()) { return new ApplicationEntityReader(context, dataToRetrieve); } else if (TimelineEntityType. - YARN_FLOW_RUN.matches(context.getEntityType())) { + YARN_FLOW_RUN == context.getSystemEntityType()) { return new FlowRunEntityReader(context, dataToRetrieve); } else if (TimelineEntityType. - YARN_FLOW_ACTIVITY.matches(context.getEntityType())) { + YARN_FLOW_ACTIVITY == context.getSystemEntityType()) { return new FlowActivityEntityReader(context, dataToRetrieve); } else { // assume we're dealing with a generic entity read @@ -73,13 +73,13 @@ public static TimelineEntityReader createMultipleEntitiesReader( TimelineDataToRetrieve dataToRetrieve) { // currently the types that are handled separate from the generic entity // table are application, flow run, and flow activity entities - if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) { + if (TimelineEntityType.YARN_APPLICATION == context.getSystemEntityType()) { return new ApplicationEntityReader(context, filters, dataToRetrieve); } else if (TimelineEntityType. - YARN_FLOW_ACTIVITY.matches(context.getEntityType())) { + YARN_FLOW_ACTIVITY == context.getSystemEntityType()) { return new FlowActivityEntityReader(context, filters, dataToRetrieve); } else if (TimelineEntityType. - YARN_FLOW_RUN.matches(context.getEntityType())) { + YARN_FLOW_RUN == context.getSystemEntityType()) { return new FlowRunEntityReader(context, filters, dataToRetrieve); } else { if (context.getDoAsUser() != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java index 67c3d29..52342a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java @@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.server.timelineservice.TimelineContext; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; /** * Encapsulates fields necessary to make a query in timeline reader. @@ -33,6 +34,8 @@ private String entityId; private Long entityIdPrefix; private String doAsUser; + private TimelineEntityType systemEntityType; + public TimelineReaderContext(String clusterId, String userId, String flowName, Long flowRunId, String appId, String entityType, String entityId) { super(clusterId, userId, flowName, flowRunId, appId); @@ -55,10 +58,19 @@ public TimelineReaderContext(String clusterId, String userId, String flowName, this.doAsUser = doasUser; } + public TimelineReaderContext(String clusterId, String userId, String flowName, + Long flowRunId, String appId, String entityType, Long entityIdPrefix, + String entityId, String doasUser, TimelineEntityType systemEntityType) { + this(clusterId, userId, flowName, flowRunId, appId, entityType, + entityIdPrefix, entityId, doasUser); + this.systemEntityType = systemEntityType; + } + public TimelineReaderContext(TimelineReaderContext other) { this(other.getClusterId(), other.getUserId(), other.getFlowName(), other.getFlowRunId(), other.getAppId(), other.getEntityType(), - other.getEntityIdPrefix(), other.getEntityId(), other.getDoAsUser()); + other.getEntityIdPrefix(), other.getEntityId(), other.getDoAsUser(), + other.getSystemEntityType()); } @Override @@ -70,6 +82,9 @@ public int hashCode() { result = prime * result + ((entityId == null) ? 0 : entityId.hashCode()); result = prime * result + ((entityType == null) ? 0 : entityType.hashCode()); + result = + prime * result + ((systemEntityType == null) ? 0 : + systemEntityType.hashCode()); return result; } @@ -96,6 +111,13 @@ public boolean equals(Object obj) { } else if (!entityType.equals(other.entityType)) { return false; } + if (systemEntityType == null) { + if (other.systemEntityType != null) { + return false; + } + } else if (systemEntityType != other.systemEntityType) { + return false; + } return true; } @@ -130,4 +152,13 @@ public String getDoAsUser() { public void setDoAsUser(String doAsUser) { this.doAsUser = doAsUser; } -} \ No newline at end of file + + public TimelineEntityType getSystemEntityType() { + return systemEntityType; + } + + public void setSystemEntityType(TimelineEntityType systemEntityType) { + this.systemEntityType = systemEntityType; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index 330e1f4..a0a5cd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -338,6 +338,7 @@ public TimelineAbout about( } context.setEntityType( TimelineReaderWebServicesUtils.parseStr(entityType)); + context.setSystemEntityType(null); entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, @@ -346,7 +347,7 @@ public TimelineAbout about( TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( confsToRetrieve, metricsToRetrieve, fields, metricsLimit, metricsTimeStart, metricsTimeEnd)); - checkAccessForGenericEntities(entities, callerUGI, entityType); + checkAccessForGenericEntities(entities, callerUGI, null); succeeded = true; } catch (Exception e) { handleException(e, url, startTime, @@ -483,7 +484,7 @@ public TimelineAbout about( flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, null); } /** @@ -603,6 +604,24 @@ public TimelineAbout about( @QueryParam("metricstimestart") String metricsTimeStart, @QueryParam("metricstimeend") String metricsTimeEnd, @QueryParam("fromid") String fromId) { + return getEntities(req, res, null, appId, entityType, userId, flowName, + flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, + isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId, null); + } + + + public Set getEntities(HttpServletRequest req, + HttpServletResponse res, String clusterId, String appId, + String entityType, String userId, String flowName, + String flowRunId, String limit, String createdTimeStart, + String createdTimeEnd, String relatesTo, String isRelatedTo, + String infofilters, String conffilters, String metricfilters, + String eventfilters, String confsToRetrieve, String metricsToRetrieve, + String fields, String metricsLimit, String metricsTimeStart, + String metricsTimeEnd, String fromId, + TimelineEntityType systemEntityType) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -618,7 +637,7 @@ public TimelineAbout about( try { TimelineReaderContext context = TimelineReaderWebServicesUtils .createTimelineReaderContext(clusterId, userId, flowName, flowRunId, - appId, entityType, null, null); + appId, entityType, null, null, systemEntityType); entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils .createTimelineEntityFilters(limit, createdTimeStart, @@ -628,7 +647,7 @@ public TimelineAbout about( .createTimelineDataToRetrieve(confsToRetrieve, metricsToRetrieve, fields, metricsLimit, metricsTimeStart, metricsTimeEnd)); - checkAccessForGenericEntities(entities, callerUGI, entityType); + checkAccessForGenericEntities(entities, callerUGI, systemEntityType); succeeded = true; } catch (Exception e) { handleException(e, url, startTime, @@ -720,6 +739,7 @@ public TimelineEntity getEntity( if (context == null) { throw new BadRequestException("Incorrect UID " + uId); } + context.setSystemEntityType(null); entity = timelineReaderManager.getEntity(context, TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( confsToRetrieve, metricsToRetrieve, fields, metricsLimit, @@ -820,7 +840,7 @@ public TimelineEntity getEntity( @QueryParam("entityidprefix") String entityIdPrefix) { return getEntity(req, res, null, appId, entityType, entityId, userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, - metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix); + metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, null); } /** @@ -899,6 +919,18 @@ public TimelineEntity getEntity( @QueryParam("metricstimestart") String metricsTimeStart, @QueryParam("metricstimeend") String metricsTimeEnd, @QueryParam("entityidprefix") String entityIdPrefix) { + return getEntity(req, res, clusterId, appId, entityType, entityId, + userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, + metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, null); + } + + public TimelineEntity getEntity(HttpServletRequest req, + HttpServletResponse res, String clusterId, String appId, + String entityType, String entityId, String userId, String flowName, + String flowRunId, String confsToRetrieve, String metricsToRetrieve, + String fields, String metricsLimit, String metricsTimeStart, + String metricsTimeEnd, String entityIdPrefix, + TimelineEntityType systemEntityType) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -915,7 +947,7 @@ public TimelineEntity getEntity( entity = timelineReaderManager.getEntity( TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, userId, flowName, flowRunId, appId, entityType, - entityIdPrefix, entityId), + entityIdPrefix, entityId, systemEntityType), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( confsToRetrieve, metricsToRetrieve, fields, metricsLimit, metricsTimeStart, metricsTimeEnd)); @@ -939,6 +971,7 @@ public TimelineEntity getEntity( return entity; } + /** * Return a single flow run for given UID which is a delimited string * containing clusterid, userid, flow name and flowrun id. @@ -991,6 +1024,7 @@ public TimelineEntity getFlowRun( // TODO to be removed or modified once ACL story is played checkAccess(timelineReaderManager, callerUGI, context.getUserId()); context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString()); + context.setSystemEntityType(TimelineEntityType.YARN_FLOW_RUN); entity = timelineReaderManager.getEntity(context, TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( null, metricsToRetrieve, null, null, null, null)); @@ -1103,7 +1137,8 @@ public TimelineEntity getFlowRun( try { TimelineReaderContext context = TimelineReaderWebServicesUtils .createTimelineReaderContext(clusterId, userId, flowName, flowRunId, - null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null); + null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, + TimelineEntityType.YARN_FLOW_RUN); // TODO to be removed or modified once ACL story is played checkAccess(timelineReaderManager, callerUGI, context.getUserId()); @@ -1205,6 +1240,7 @@ public TimelineEntity getFlowRun( // TODO to be removed or modified once ACL story is played checkAccess(timelineReaderManager, callerUGI, context.getUserId()); context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString()); + context.setSystemEntityType(TimelineEntityType.YARN_FLOW_RUN); entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, null, null, null, @@ -1359,7 +1395,7 @@ public TimelineEntity getFlowRun( TimelineReaderContext timelineReaderContext = TimelineReaderWebServicesUtils .createTimelineReaderContext(clusterId, userId, flowName, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, - null); + null, TimelineEntityType.YARN_FLOW_RUN); // TODO to be removed or modified once ACL story is played checkAccess(timelineReaderManager, callerUGI, timelineReaderContext.getUserId()); @@ -1507,7 +1543,8 @@ public TimelineEntity getFlowRun( entities = timelineReaderManager.getEntities( TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, null, null, null, null, - TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null, null), + TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null, null, + TimelineEntityType.YARN_FLOW_ACTIVITY), entityFilters, TimelineReaderWebServicesUtils. createTimelineDataToRetrieve(null, null, null, null, null, null)); succeeded = true; @@ -1603,6 +1640,7 @@ public TimelineEntity getApp( throw new BadRequestException("Incorrect UID " + uId); } context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString()); + context.setSystemEntityType(TimelineEntityType.YARN_APPLICATION); entity = timelineReaderManager.getEntity(context, TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( confsToRetrieve, metricsToRetrieve, fields, metricsLimit, @@ -1779,7 +1817,8 @@ public TimelineEntity getApp( entity = timelineReaderManager.getEntity( TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, userId, flowName, flowRunId, appId, - TimelineEntityType.YARN_APPLICATION.toString(), null, null), + TimelineEntityType.YARN_APPLICATION.toString(), null, null, + TimelineEntityType.YARN_APPLICATION), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( confsToRetrieve, metricsToRetrieve, fields, metricsLimit, metricsTimeStart, metricsTimeEnd)); @@ -1922,6 +1961,7 @@ public TimelineEntity getApp( // TODO to be removed or modified once ACL story is played checkAccess(timelineReaderManager, callerUGI, context.getUserId()); context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString()); + context.setSystemEntityType(TimelineEntityType.YARN_APPLICATION); entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, @@ -2055,7 +2095,8 @@ public TimelineEntity getApp( flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, + TimelineEntityType.YARN_APPLICATION); } /** @@ -2169,7 +2210,8 @@ public TimelineEntity getApp( flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, + TimelineEntityType.YARN_APPLICATION); } /** @@ -2277,7 +2319,8 @@ public TimelineEntity getApp( null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, + TimelineEntityType.YARN_APPLICATION); } /** @@ -2387,7 +2430,8 @@ public TimelineEntity getApp( null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, + TimelineEntityType.YARN_APPLICATION); } /** @@ -2636,7 +2680,7 @@ public TimelineEntity getApp( flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, null); } /** @@ -2801,7 +2845,7 @@ public TimelineEntity getAppAttempt(@Context HttpServletRequest req, return getEntity(req, res, clusterId, appId, TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), appAttemptId, userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, - metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix); + metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, null); } /** @@ -3068,7 +3112,7 @@ public TimelineEntity getAppAttempt(@Context HttpServletRequest req, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilter, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, - metricsTimeStart, metricsTimeEnd, fromId); + metricsTimeStart, metricsTimeEnd, fromId, null); } /** @@ -3232,7 +3276,7 @@ public TimelineEntity getContainer(@Context HttpServletRequest req, return getEntity(req, res, clusterId, appId, TimelineEntityType.YARN_CONTAINER.toString(), containerId, userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, - metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix); + metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, null); } /** @@ -3328,7 +3372,7 @@ public TimelineEntity getContainer(@Context HttpServletRequest req, results = timelineReaderManager.getEntityTypes( TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, userId, flowName, flowRunId, appId, - null, null, null)); + null, null, null, null)); succeeded = true; } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); @@ -3412,7 +3456,7 @@ public TimelineEntity getContainer(@Context HttpServletRequest req, try { TimelineReaderContext context = TimelineReaderWebServicesUtils.createTimelineReaderContext(clusterId, - null, null, null, null, entityType, null, null, userId); + null, null, null, null, entityType, null, null, userId, null); entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, @@ -3488,7 +3532,7 @@ public TimelineEntity getContainer(@Context HttpServletRequest req, try { TimelineReaderContext context = TimelineReaderWebServicesUtils .createTimelineReaderContext(clusterId, null, null, null, null, - entityType, entityIdPrefix, entityId, userId); + entityType, entityIdPrefix, entityId, userId, null); entities = timelineReaderManager.getEntities(context, new TimelineEntityFilters.Builder().build(), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( @@ -3568,7 +3612,8 @@ private void checkAccessForGenericEntity(TimelineEntity entity, // TODO to be removed or modified once ACL story is played private void checkAccessForGenericEntities(Set entities, - UserGroupInformation callerUGI, String entityType) throws Exception { + UserGroupInformation callerUGI, TimelineEntityType systemEntityType) + throws Exception { if (entities != null && entities.size() > 0 && isDisplayEntityPerUserFilterEnabled( getTimelineReaderManager().getConfig())) { @@ -3576,7 +3621,7 @@ private void checkAccessForGenericEntities(Set entities, TimelineEntity entity = entities.iterator().next(); String uid = (String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY); - if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) { + if (TimelineEntityType.YARN_APPLICATION == systemEntityType) { timelineReaderContext = TimelineFromIdConverter.APPLICATION_FROMID.decodeUID(uid); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java index bea81c7..529e4ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java @@ -26,6 +26,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; @@ -47,24 +48,31 @@ private TimelineReaderWebServicesUtils() { * @param appId App Id. * @param entityType Entity Type. * @param entityId Entity Id. + * @param systemEntityType System TimelineEntityType. * @return a {@link TimelineReaderContext} object. */ static TimelineReaderContext createTimelineReaderContext(String clusterId, String userId, String flowName, String flowRunId, String appId, - String entityType, String entityIdPrefix, String entityId) { - return new TimelineReaderContext(parseStr(clusterId), parseStr(userId), - parseStr(flowName), parseLongStr(flowRunId), parseStr(appId), - parseStr(entityType), parseLongStr(entityIdPrefix), parseStr(entityId)); + String entityType, String entityIdPrefix, String entityId, + TimelineEntityType systemEntityType) { + TimelineReaderContext context = new TimelineReaderContext( + parseStr(clusterId), parseStr(userId), parseStr(flowName), + parseLongStr(flowRunId), parseStr(appId), parseStr(entityType), + parseLongStr(entityIdPrefix), parseStr(entityId)); + context.setSystemEntityType(systemEntityType); + return context; } static TimelineReaderContext createTimelineReaderContext(String clusterId, String userId, String flowName, String flowRunId, String appId, String entityType, String entityIdPrefix, String entityId, - String doAsUser) { - return new TimelineReaderContext(parseStr(clusterId), parseStr(userId), - parseStr(flowName), parseLongStr(flowRunId), parseStr(appId), - parseStr(entityType), parseLongStr(entityIdPrefix), parseStr(entityId), - parseStr(doAsUser)); + String doAsUser, TimelineEntityType systemEntityType) { + TimelineReaderContext context = new TimelineReaderContext( + parseStr(clusterId), parseStr(userId), parseStr(flowName), + parseLongStr(flowRunId), parseStr(appId), parseStr(entityType), + parseLongStr(entityIdPrefix), parseStr(entityId), parseStr(doAsUser)); + context.setSystemEntityType(systemEntityType); + return context; } /** -- 2.7.4 (Apple Git-66)