diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8752e5d..e45bfe3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2047,6 +2047,18 @@ public static boolean isAclEnabled(Configuration conf) { + "hbase.coprocessor.app-final-value-retention-milliseconds"; /** + * The name of the setting for the location of the coprocessor + * jar on hdfs. + */ + public static final String FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION = + TIMELINE_SERVICE_PREFIX + + "hbase.coprocessor.jar.hdfs.location"; + + /** default hdfs location for flowrun coprocessor jar. */ + public static final String DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR = + "/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar"; + + /** * The name for setting that points to an optional HBase configuration * (hbase-site.xml file) with settings that will override the ones found on * the classpath. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 6f62fd8..16954a0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2243,6 +2243,15 @@ + The default hdfs location for flowrun coprocessor jar. + + yarn.timeline-service.hbase.coprocessor.jar.hdfs.location + + /hbase/coprocessor/hadoop-yarn-server-timelineservice.jar + + + + The value of this parameter sets the prefix for all tables that are part of timeline service in the hbase storage schema. It can be set to "dev." or "staging." if it is to be used for development or staging instances. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java index 1b66fcb..b3f68be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java @@ -50,8 +50,8 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator; import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.junit.After; @@ -86,7 +86,7 @@ public static void setup() throws Exception { Configuration conf = util.getConfiguration(); conf.setInt("hfile.format.version", 3); util.startMiniCluster(); - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); + DataGeneratorForTest.createSchema(conf); loadData(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java index b56a752..532881a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; @@ -31,10 +32,33 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; -final class DataGeneratorForTest { - static void loadApps(HBaseTestingUtility util) throws IOException { +/** + * Utility class that creates the schema and generates test data. + */ +public final class DataGeneratorForTest { + + // private constructor for utility class + private DataGeneratorForTest() { + } + + /** + * Creates the schema for timeline service. + * @param conf + * @throws IOException + */ + public static void createSchema(final Configuration conf) + throws IOException { + // set the jar location to null so that + // the coprocessor class is loaded from classpath + conf.set(YarnConfiguration.FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION, " "); + // now create all tables + TimelineSchemaCreator.createAllTables(conf, false); + } + + public static void loadApps(HBaseTestingUtility util) throws IOException { TimelineEntities te = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); String id = "application_1111111111_2222"; @@ -43,11 +67,7 @@ static void loadApps(HBaseTestingUtility util) throws IOException { Long cTime = 1425016502000L; entity.setCreatedTime(cTime); // add the info map in Timeline Entity - Map infoMap = new HashMap<>(); - infoMap.put("infoMapKey1", "infoMapValue2"); - infoMap.put("infoMapKey2", 20); - infoMap.put("infoMapKey3", 85.85); - entity.addInfo(infoMap); + entity.addInfo(getInfoMap3()); // add the isRelatedToEntity info Set isRelatedToSet = new HashSet<>(); isRelatedToSet.add("relatedto1"); @@ -72,29 +92,15 @@ static void loadApps(HBaseTestingUtility util) throws IOException { entity.addConfigs(conf); // add metrics Set metrics = new HashSet<>(); - TimelineMetric m1 = new TimelineMetric(); - m1.setId("MAP_SLOT_MILLIS"); - Map metricValues = new HashMap<>(); long ts = System.currentTimeMillis(); - metricValues.put(ts - 120000, 100000000); - metricValues.put(ts - 100000, 200000000); - metricValues.put(ts - 80000, 300000000); - metricValues.put(ts - 60000, 400000000); - metricValues.put(ts - 40000, 50000000000L); - metricValues.put(ts - 20000, 60000000000L); - m1.setType(Type.TIME_SERIES); - m1.setValues(metricValues); - metrics.add(m1); + metrics.add(getMetric4(ts)); TimelineMetric m12 = new TimelineMetric(); m12.setId("MAP1_BYTES"); m12.addValue(ts, 50); metrics.add(m12); entity.addMetrics(metrics); - TimelineEvent event = new TimelineEvent(); - event.setId("start_event"); - event.setTimestamp(ts); - entity.addEvent(event); + entity.addEvent(addStartEvent(ts)); te.addEntity(entity); TimelineEntities te1 = new TimelineEntities(); TimelineEntity entity1 = new TimelineEntity(); @@ -103,10 +109,7 @@ static void loadApps(HBaseTestingUtility util) throws IOException { entity1.setType(TimelineEntityType.YARN_APPLICATION.toString()); entity1.setCreatedTime(cTime + 20L); // add the info map in Timeline Entity - Map infoMap1 = new HashMap<>(); - infoMap1.put("infoMapKey1", "infoMapValue1"); - infoMap1.put("infoMapKey2", 10); - entity1.addInfo(infoMap1); + entity1.addInfo(getInfoMap4()); // add the isRelatedToEntity info Set isRelatedToSet1 = new HashSet<>(); @@ -134,21 +137,7 @@ static void loadApps(HBaseTestingUtility util) throws IOException { entity1.addConfigs(conf1); // add metrics - Set metrics1 = new HashSet<>(); - TimelineMetric m2 = new TimelineMetric(); - m2.setId("MAP1_SLOT_MILLIS"); - Map metricValues1 = new HashMap<>(); - long ts1 = System.currentTimeMillis(); - metricValues1.put(ts1 - 120000, 100000000); - metricValues1.put(ts1 - 100000, 200000000); - metricValues1.put(ts1 - 80000, 300000000); - metricValues1.put(ts1 - 60000, 400000000); - metricValues1.put(ts1 - 40000, 50000000000L); - metricValues1.put(ts1 - 20000, 60000000000L); - m2.setType(Type.TIME_SERIES); - m2.setValues(metricValues1); - metrics1.add(m2); - entity1.addMetrics(metrics1); + entity1.addMetrics(getMetrics4()); TimelineEvent event11 = new TimelineEvent(); event11.setId("end_event"); event11.setTimestamp(ts); @@ -160,27 +149,7 @@ static void loadApps(HBaseTestingUtility util) throws IOException { te1.addEntity(entity1); TimelineEntities te2 = new TimelineEntities(); - TimelineEntity entity2 = new TimelineEntity(); - String id2 = "application_1111111111_4444"; - entity2.setId(id2); - entity2.setType(TimelineEntityType.YARN_APPLICATION.toString()); - entity2.setCreatedTime(cTime + 40L); - TimelineEvent event21 = new TimelineEvent(); - event21.setId("update_event"); - event21.setTimestamp(ts - 20); - entity2.addEvent(event21); - Set isRelatedToSet2 = new HashSet(); - isRelatedToSet2.add("relatedto3"); - Map> isRelatedTo2 = new HashMap<>(); - isRelatedTo2.put("task1", isRelatedToSet2); - entity2.setIsRelatedToEntities(isRelatedTo2); - Map> relatesTo3 = new HashMap<>(); - Set relatesToSet14 = new HashSet(); - relatesToSet14.add("relatesto7"); - relatesTo3.put("container2", relatesToSet14); - entity2.setRelatesToEntities(relatesTo3); - - te2.addEntity(entity2); + te2.addEntity(getEntity4(cTime, ts)); HBaseTimelineWriterImpl hbi = null; try { hbi = new HBaseTimelineWriterImpl(util.getConfiguration()); @@ -206,29 +175,86 @@ static void loadApps(HBaseTestingUtility util) throws IOException { } } - static void loadEntities(HBaseTestingUtility util) throws IOException { - TimelineEntities te = new TimelineEntities(); - TimelineEntity entity = new TimelineEntity(); - String id = "hello"; - String type = "world"; - entity.setId(id); - entity.setType(type); - Long cTime = 1425016502000L; - entity.setCreatedTime(cTime); - // add the info map in Timeline Entity + private static Set getMetrics4() { + Set metrics1 = new HashSet<>(); + TimelineMetric m2 = new TimelineMetric(); + m2.setId("MAP1_SLOT_MILLIS"); + long ts1 = System.currentTimeMillis(); + Map metricValues1 = new HashMap<>(); + metricValues1.put(ts1 - 120000, 100000000); + metricValues1.put(ts1 - 100000, 200000000); + metricValues1.put(ts1 - 80000, 300000000); + metricValues1.put(ts1 - 60000, 400000000); + metricValues1.put(ts1 - 40000, 50000000000L); + metricValues1.put(ts1 - 20000, 60000000000L); + m2.setType(Type.TIME_SERIES); + m2.setValues(metricValues1); + metrics1.add(m2); + return metrics1; + } + + private static TimelineEntity getEntity4(long cTime, long ts) { + TimelineEntity entity2 = new TimelineEntity(); + String id2 = "application_1111111111_4444"; + entity2.setId(id2); + entity2.setType(TimelineEntityType.YARN_APPLICATION.toString()); + entity2.setCreatedTime(cTime + 40L); + TimelineEvent event21 = new TimelineEvent(); + event21.setId("update_event"); + event21.setTimestamp(ts - 20); + entity2.addEvent(event21); + Set isRelatedToSet2 = new HashSet(); + isRelatedToSet2.add("relatedto3"); + Map> isRelatedTo2 = new HashMap<>(); + isRelatedTo2.put("task1", isRelatedToSet2); + entity2.setIsRelatedToEntities(isRelatedTo2); + Map> relatesTo3 = new HashMap<>(); + Set relatesToSet14 = new HashSet(); + relatesToSet14.add("relatesto7"); + relatesTo3.put("container2", relatesToSet14); + entity2.setRelatesToEntities(relatesTo3); + return entity2; + } + + private static Map getInfoMap4() { + Map infoMap1 = new HashMap<>(); + infoMap1.put("infoMapKey1", "infoMapValue1"); + infoMap1.put("infoMapKey2", 10); + return infoMap1; + } + + private static TimelineMetric getMetric4(long ts) { + TimelineMetric m1 = new TimelineMetric(); + m1.setId("MAP_SLOT_MILLIS"); + Map metricValues = new HashMap<>(); + metricValues.put(ts - 120000, 100000000); + metricValues.put(ts - 100000, 200000000); + metricValues.put(ts - 80000, 300000000); + metricValues.put(ts - 60000, 400000000); + metricValues.put(ts - 40000, 50000000000L); + metricValues.put(ts - 20000, 60000000000L); + m1.setType(Type.TIME_SERIES); + m1.setValues(metricValues); + return m1; + } + + private static Map getInfoMap3() { + Map infoMap = new HashMap<>(); + infoMap.put("infoMapKey1", "infoMapValue2"); + infoMap.put("infoMapKey2", 20); + infoMap.put("infoMapKey3", 85.85); + return infoMap; + } + + private static Map getInfoMap1() { Map infoMap = new HashMap<>(); infoMap.put("infoMapKey1", "infoMapValue2"); infoMap.put("infoMapKey2", 20); infoMap.put("infoMapKey3", 71.4); - entity.addInfo(infoMap); - // add the isRelatedToEntity info - Set isRelatedToSet = new HashSet<>(); - isRelatedToSet.add("relatedto1"); - Map> isRelatedTo = new HashMap<>(); - isRelatedTo.put("task", isRelatedToSet); - entity.setIsRelatedToEntities(isRelatedTo); + return infoMap; + } - // add the relatesTo info + private static Map> getRelatesTo1() { Set relatesToSet = new HashSet(); relatesToSet.add("relatesto1"); relatesToSet.add("relatesto3"); @@ -237,29 +263,78 @@ static void loadEntities(HBaseTestingUtility util) throws IOException { Set relatesToSet11 = new HashSet<>(); relatesToSet11.add("relatesto4"); relatesTo.put("container1", relatesToSet11); - entity.setRelatesToEntities(relatesTo); + return relatesTo; + } - // add some config entries + private static Map getConfig1() { Map conf = new HashMap<>(); conf.put("config_param1", "value1"); conf.put("config_param2", "value2"); conf.put("cfg_param1", "value3"); - entity.addConfigs(conf); + return conf; + } - // add metrics - Set metrics = new HashSet<>(); - TimelineMetric m1 = new TimelineMetric(); - m1.setId("MAP_SLOT_MILLIS"); + private static Map getConfig2() { + Map conf1 = new HashMap<>(); + conf1.put("cfg_param1", "value1"); + conf1.put("cfg_param2", "value2"); + return conf1; + } + + private static Map getInfoMap2() { + Map infoMap1 = new HashMap<>(); + infoMap1.put("infoMapKey1", "infoMapValue1"); + infoMap1.put("infoMapKey2", 10); + return infoMap1; + } + + private static Map> getIsRelatedTo1() { + Set isRelatedToSet = new HashSet<>(); + isRelatedToSet.add("relatedto1"); + Map> isRelatedTo = new HashMap<>(); + isRelatedTo.put("task", isRelatedToSet); + return isRelatedTo; + } + + private static Map getMetricValues1(long ts) { Map metricValues = new HashMap<>(); - long ts = System.currentTimeMillis(); metricValues.put(ts - 120000, 100000000); metricValues.put(ts - 100000, 200000000); metricValues.put(ts - 80000, 300000000); metricValues.put(ts - 60000, 400000000); metricValues.put(ts - 40000, 50000000000L); metricValues.put(ts - 20000, 70000000000L); + return metricValues; + } + + public static void loadEntities(HBaseTestingUtility util) + throws IOException { + TimelineEntities te = new TimelineEntities(); + TimelineEntity entity = new TimelineEntity(); + String id = "hello"; + String type = "world"; + entity.setId(id); + entity.setType(type); + Long cTime = 1425016502000L; + entity.setCreatedTime(cTime); + // add the info map in Timeline Entity + entity.addInfo(getInfoMap1()); + // add the isRelatedToEntity info + entity.setIsRelatedToEntities(getIsRelatedTo1()); + + // add the relatesTo info + entity.setRelatesToEntities(getRelatesTo1()); + + // add some config entries + entity.addConfigs(getConfig1()); + + // add metrics + Set metrics = new HashSet<>(); + TimelineMetric m1 = new TimelineMetric(); + m1.setId("MAP_SLOT_MILLIS"); + long ts = System.currentTimeMillis(); m1.setType(Type.TIME_SERIES); - m1.setValues(metricValues); + m1.setValues(getMetricValues1(ts)); metrics.add(m1); TimelineMetric m12 = new TimelineMetric(); @@ -267,10 +342,7 @@ static void loadEntities(HBaseTestingUtility util) throws IOException { m12.addValue(ts, 50); metrics.add(m12); entity.addMetrics(metrics); - TimelineEvent event = new TimelineEvent(); - event.setId("start_event"); - event.setTimestamp(ts); - entity.addEvent(event); + entity.addEvent(addStartEvent(ts)); te.addEntity(entity); TimelineEntity entity1 = new TimelineEntity(); @@ -280,10 +352,7 @@ static void loadEntities(HBaseTestingUtility util) throws IOException { entity1.setCreatedTime(cTime + 20L); // add the info map in Timeline Entity - Map infoMap1 = new HashMap<>(); - infoMap1.put("infoMapKey1", "infoMapValue1"); - infoMap1.put("infoMapKey2", 10); - entity1.addInfo(infoMap1); + entity1.addInfo(getInfoMap2()); // add event. TimelineEvent event11 = new TimelineEvent(); @@ -297,15 +366,7 @@ static void loadEntities(HBaseTestingUtility util) throws IOException { // add the isRelatedToEntity info - Set isRelatedToSet1 = new HashSet<>(); - isRelatedToSet1.add("relatedto3"); - isRelatedToSet1.add("relatedto5"); - Map> isRelatedTo1 = new HashMap<>(); - isRelatedTo1.put("task1", isRelatedToSet1); - Set isRelatedToSet11 = new HashSet<>(); - isRelatedToSet11.add("relatedto4"); - isRelatedTo1.put("task2", isRelatedToSet11); - entity1.setIsRelatedToEntities(isRelatedTo1); + entity1.setIsRelatedToEntities(getIsRelatedTo2()); // add the relatesTo info Set relatesToSet1 = new HashSet(); @@ -316,49 +377,20 @@ static void loadEntities(HBaseTestingUtility util) throws IOException { entity1.setRelatesToEntities(relatesTo1); // add some config entries - Map conf1 = new HashMap<>(); - conf1.put("cfg_param1", "value1"); - conf1.put("cfg_param2", "value2"); - entity1.addConfigs(conf1); + entity1.addConfigs(getConfig2()); // add metrics Set metrics1 = new HashSet<>(); TimelineMetric m2 = new TimelineMetric(); m2.setId("MAP1_SLOT_MILLIS"); - Map metricValues1 = new HashMap<>(); long ts1 = System.currentTimeMillis(); - metricValues1.put(ts1 - 120000, 100000000); - metricValues1.put(ts1 - 100000, 200000000); - metricValues1.put(ts1 - 80000, 300000000); - metricValues1.put(ts1 - 60000, 400000000); - metricValues1.put(ts1 - 40000, 50000000000L); - metricValues1.put(ts1 - 20000, 60000000000L); m2.setType(Type.TIME_SERIES); - m2.setValues(metricValues1); + m2.setValues(getMetricValues2(ts1)); metrics1.add(m2); entity1.addMetrics(metrics1); te.addEntity(entity1); - TimelineEntity entity2 = new TimelineEntity(); - String id2 = "hello2"; - entity2.setId(id2); - entity2.setType(type); - entity2.setCreatedTime(cTime + 40L); - TimelineEvent event21 = new TimelineEvent(); - event21.setId("update_event"); - event21.setTimestamp(ts - 20); - entity2.addEvent(event21); - Set isRelatedToSet2 = new HashSet<>(); - isRelatedToSet2.add("relatedto3"); - Map> isRelatedTo2 = new HashMap<>(); - isRelatedTo2.put("task1", isRelatedToSet2); - entity2.setIsRelatedToEntities(isRelatedTo2); - Map> relatesTo3 = new HashMap<>(); - Set relatesToSet14 = new HashSet<>(); - relatesToSet14.add("relatesto7"); - relatesTo3.put("container2", relatesToSet14); - entity2.setRelatesToEntities(relatesTo3); - te.addEntity(entity2); + te.addEntity(getEntity2(type, cTime, ts)); // For listing types for (int i = 0; i < 10; i++) { @@ -421,4 +453,58 @@ static void loadEntities(HBaseTestingUtility util) throws IOException { } } } + + private static TimelineEntity getEntity2(String type, long cTime, + long ts) { + TimelineEntity entity2 = new TimelineEntity(); + String id2 = "hello2"; + entity2.setId(id2); + entity2.setType(type); + entity2.setCreatedTime(cTime + 40L); + TimelineEvent event21 = new TimelineEvent(); + event21.setId("update_event"); + event21.setTimestamp(ts - 20); + entity2.addEvent(event21); + Set isRelatedToSet2 = new HashSet<>(); + isRelatedToSet2.add("relatedto3"); + Map> isRelatedTo2 = new HashMap<>(); + isRelatedTo2.put("task1", isRelatedToSet2); + entity2.setIsRelatedToEntities(isRelatedTo2); + Map> relatesTo3 = new HashMap<>(); + Set relatesToSet14 = new HashSet<>(); + relatesToSet14.add("relatesto7"); + relatesTo3.put("container2", relatesToSet14); + entity2.setRelatesToEntities(relatesTo3); + return entity2; + } + + private static TimelineEvent addStartEvent(long ts) { + TimelineEvent event = new TimelineEvent(); + event.setId("start_event"); + event.setTimestamp(ts); + return event; + } + + private static Map getMetricValues2(long ts1) { + Map metricValues1 = new HashMap<>(); + metricValues1.put(ts1 - 120000, 100000000); + metricValues1.put(ts1 - 100000, 200000000); + metricValues1.put(ts1 - 80000, 300000000); + metricValues1.put(ts1 - 60000, 400000000); + metricValues1.put(ts1 - 40000, 50000000000L); + metricValues1.put(ts1 - 20000, 60000000000L); + return metricValues1; + } + + private static Map> getIsRelatedTo2() { + Set isRelatedToSet1 = new HashSet<>(); + isRelatedToSet1.add("relatedto3"); + isRelatedToSet1.add("relatedto5"); + Map> isRelatedTo1 = new HashMap<>(); + isRelatedTo1.put("task1", isRelatedToSet1); + Set isRelatedToSet11 = new HashSet<>(); + isRelatedToSet11.add("relatedto4"); + isRelatedTo1.put("task2", isRelatedToSet11); + return isRelatedTo1; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java index e70198a..1aff2e0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java @@ -91,14 +91,10 @@ public static void setupBeforeClass() throws Exception { util = new HBaseTestingUtility(); util.startMiniCluster(); - createSchema(); + DataGeneratorForTest.createSchema(util.getConfiguration()); DataGeneratorForTest.loadApps(util); } - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); - } - @Before public void init() throws Exception { reader = new HBaseTimelineReaderImpl(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java index 9b35ef3..c731631 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java @@ -98,14 +98,10 @@ public static void setupBeforeClass() throws Exception { util = new HBaseTestingUtility(); util.startMiniCluster(); - createSchema(); + DataGeneratorForTest.createSchema(util.getConfiguration()); DataGeneratorForTest.loadEntities(util); } - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); - } - @Before public void init() throws Exception { reader = new HBaseTimelineReaderImpl(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java index 53045e5..0dcd171 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java @@ -48,17 +48,15 @@ @BeforeClass public static void setupBeforeClass() throws Exception { util = new HBaseTestingUtility(); + Configuration conf = util.getConfiguration(); + conf.setInt("hfile.format.version", 3); util.startMiniCluster(); } - private static void createSchema(Configuration conf) throws IOException { - TimelineSchemaCreator.createAllTables(conf, false); - } - @Test public void createWithDefaultPrefix() throws IOException { Configuration hbaseConf = util.getConfiguration(); - createSchema(hbaseConf); + DataGeneratorForTest.createSchema(hbaseConf); Connection conn = null; conn = ConnectionFactory.createConnection(hbaseConf); Admin admin = conn.getAdmin(); @@ -88,7 +86,7 @@ public void createWithSetPrefix() throws IOException { String prefix = "unit-test."; hbaseConf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME, prefix); - createSchema(hbaseConf); + DataGeneratorForTest.createSchema(hbaseConf); Connection conn = null; conn = ConnectionFactory.createConnection(hbaseConf); Admin admin = conn.getAdmin(); @@ -115,7 +113,7 @@ public void createWithSetPrefix() throws IOException { prefix = "yet-another-unit-test."; hbaseConf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME, prefix); - createSchema(hbaseConf); + DataGeneratorForTest.createSchema(hbaseConf); entityTableName = BaseTable.getTableName(hbaseConf, EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME); assertTrue(admin.tableExists(entityTableName)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java index b608987..8ab32df 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java @@ -33,7 +33,7 @@ /** * Generates the data/entities for the FlowRun and FlowActivity Tables. */ -final class TestFlowDataGenerator { +public final class TestFlowDataGenerator { private TestFlowDataGenerator() { } @@ -41,7 +41,8 @@ private TestFlowDataGenerator() { private static final String METRIC_2 = "HDFS_BYTES_READ"; public static final long END_TS_INCR = 10000L; - static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) { + public static TimelineEntity getEntityMetricsApp1(long insertTs, + Configuration c1) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunMetrics_test"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -83,7 +84,7 @@ static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) { } - static TimelineEntity getEntityMetricsApp1Complete(long insertTs, + public static TimelineEntity getEntityMetricsApp1Complete(long insertTs, Configuration c1) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunMetrics_test"; @@ -125,7 +126,7 @@ static TimelineEntity getEntityMetricsApp1Complete(long insertTs, } - static TimelineEntity getEntityMetricsApp1(long insertTs) { + public static TimelineEntity getEntityMetricsApp1(long insertTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunMetrics_test"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -168,8 +169,7 @@ static TimelineEntity getEntityMetricsApp1(long insertTs) { return entity; } - - static TimelineEntity getEntityMetricsApp2(long insertTs) { + public static TimelineEntity getEntityMetricsApp2(long insertTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunMetrics_test"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -200,7 +200,7 @@ static TimelineEntity getEntityMetricsApp2(long insertTs) { return entity; } - static TimelineEntity getEntity1() { + public static TimelineEntity getEntity1() { TimelineEntity entity = new TimelineEntity(); String id = "flowRunHello"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -243,7 +243,7 @@ static TimelineEntity getEntity1() { return entity; } - static TimelineEntity getAFullEntity(long ts, long endTs) { + public static TimelineEntity getAFullEntity(long ts, long endTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunFullEntity"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -292,7 +292,7 @@ static TimelineEntity getAFullEntity(long ts, long endTs) { return entity; } - static TimelineEntity getEntityGreaterStartTime(long startTs) { + public static TimelineEntity getEntityGreaterStartTime(long startTs) { TimelineEntity entity = new TimelineEntity(); entity.setCreatedTime(startTs); entity.setId("flowRunHello with greater start time"); @@ -308,7 +308,7 @@ static TimelineEntity getEntityGreaterStartTime(long startTs) { return entity; } - static TimelineEntity getEntityMaxEndTime(long endTs) { + public static TimelineEntity getEntityMaxEndTime(long endTs) { TimelineEntity entity = new TimelineEntity(); entity.setId("flowRunHello Max End time"); entity.setType(TimelineEntityType.YARN_APPLICATION.toString()); @@ -322,7 +322,7 @@ static TimelineEntity getEntityMaxEndTime(long endTs) { return entity; } - static TimelineEntity getEntityMinStartTime(long startTs) { + public static TimelineEntity getEntityMinStartTime(long startTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunHelloMInStartTime"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -336,7 +336,7 @@ static TimelineEntity getEntityMinStartTime(long startTs) { return entity; } - static TimelineEntity getMinFlushEntity(long startTs) { + public static TimelineEntity getMinFlushEntity(long startTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunHelloFlushEntityMin"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -350,7 +350,7 @@ static TimelineEntity getMinFlushEntity(long startTs) { return entity; } - static TimelineEntity getMaxFlushEntity(long startTs) { + public static TimelineEntity getMaxFlushEntity(long startTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunHelloFlushEntityMax"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -365,7 +365,7 @@ static TimelineEntity getMaxFlushEntity(long startTs) { return entity; } - static TimelineEntity getFlowApp1(long appCreatedTime) { + public static TimelineEntity getFlowApp1(long appCreatedTime) { TimelineEntity entity = new TimelineEntity(); String id = "flowActivity_test"; String type = TimelineEntityType.YARN_APPLICATION.toString(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java index 2db01a6..c21e874 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java @@ -47,9 +47,9 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator; import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; @@ -70,11 +70,7 @@ public static void setupBeforeClass() throws Exception { Configuration conf = util.getConfiguration(); conf.setInt("hfile.format.version", 3); util.startMiniCluster(); - createSchema(); - } - - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); + DataGeneratorForTest.createSchema(util.getConfiguration()); } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java index c066a1f..2a101cf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.flow; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -41,8 +40,8 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; @@ -57,10 +56,10 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter; +import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator; import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable; @@ -84,11 +83,7 @@ public static void setupBeforeClass() throws Exception { Configuration conf = util.getConfiguration(); conf.setInt("hfile.format.version", 3); util.startMiniCluster(); - createSchema(); - } - - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); + DataGeneratorForTest.createSchema(util.getConfiguration()); } @Test @@ -106,12 +101,7 @@ public void checkCoProcessorOff() throws IOException, InterruptedException { // check the regions. // check in flow run table util.waitUntilAllRegionsAssigned(table); - HRegionServer server = util.getRSForFirstRegionInTable(table); - List regions = server.getOnlineRegions(table); - for (Region region : regions) { - assertTrue(FlowRunTable.isFlowRunTable(region.getRegionInfo(), - hbaseConf)); - } + checkCoprocessorExists(table, true); } table = BaseTable.getTableName(hbaseConf, @@ -121,12 +111,7 @@ public void checkCoProcessorOff() throws IOException, InterruptedException { // check the regions. // check in flow activity table util.waitUntilAllRegionsAssigned(table); - HRegionServer server = util.getRSForFirstRegionInTable(table); - List regions = server.getOnlineRegions(table); - for (Region region : regions) { - assertFalse(FlowRunTable.isFlowRunTable(region.getRegionInfo(), - hbaseConf)); - } + checkCoprocessorExists(table, false); } table = BaseTable.getTableName(hbaseConf, EntityTable.TABLE_NAME_CONF_NAME, @@ -135,12 +120,23 @@ public void checkCoProcessorOff() throws IOException, InterruptedException { // check the regions. // check in entity run table util.waitUntilAllRegionsAssigned(table); - HRegionServer server = util.getRSForFirstRegionInTable(table); - List regions = server.getOnlineRegions(table); - for (Region region : regions) { - assertFalse(FlowRunTable.isFlowRunTable(region.getRegionInfo(), - hbaseConf)); + checkCoprocessorExists(table, false); + } + } + + private void checkCoprocessorExists(TableName table, boolean exists) + throws IOException, InterruptedException { + HRegionServer server = util.getRSForFirstRegionInTable(table); + List regions = server.getOnlineRegions(table); + for (Region region : regions) { + boolean found = false; + Set coprocs = region.getCoprocessorHost().getCoprocessors(); + for (String coprocName : coprocs) { + if (coprocName.contains("FlowRunCoprocessor")) { + found = true; + } } + assertEquals(found, exists); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java index 644e31a..7f783b8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java @@ -50,8 +50,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator; import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; @@ -82,11 +82,7 @@ public static void setupBeforeClass() throws Exception { Configuration conf = util.getConfiguration(); conf.setInt("hfile.format.version", 3); util.startMiniCluster(); - createSchema(); - } - - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); + DataGeneratorForTest.createSchema(util.getConfiguration()); } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java index 122d399..24101c6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java @@ -57,7 +57,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver { private static final Log LOG = LogFactory.getLog(FlowRunCoprocessor.class); - private boolean isFlowRunRegion = false; private Region region; /** @@ -71,15 +70,9 @@ public void start(CoprocessorEnvironment e) throws IOException { if (e instanceof RegionCoprocessorEnvironment) { RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e; this.region = env.getRegion(); - isFlowRunRegion = FlowRunTable.isFlowRunTable( - region.getRegionInfo(), env.getConfiguration()); } } - public boolean isFlowRunRegion() { - return isFlowRunRegion; - } - /* * (non-Javadoc) * @@ -99,10 +92,6 @@ public boolean isFlowRunRegion() { public void prePut(ObserverContext e, Put put, WALEdit edit, Durability durability) throws IOException { Map attributes = put.getAttributesMap(); - - if (!isFlowRunRegion) { - return; - } // Assumption is that all the cells in a put are the same operation. List tags = new ArrayList<>(); if ((attributes != null) && (attributes.size() > 0)) { @@ -170,10 +159,6 @@ private long getCellTimestamp(long timestamp, List tags) { @Override public void preGetOp(ObserverContext e, Get get, List results) throws IOException { - if (!isFlowRunRegion) { - return; - } - Scan scan = new Scan(get); scan.setMaxVersions(); RegionScanner scanner = null; @@ -205,12 +190,9 @@ public void preGetOp(ObserverContext e, public RegionScanner preScannerOpen( ObserverContext e, Scan scan, RegionScanner scanner) throws IOException { - - if (isFlowRunRegion) { - // set max versions for scan to see all - // versions to aggregate for metrics - scan.setMaxVersions(); - } + // set max versions for scan to see all + // versions to aggregate for metrics + scan.setMaxVersions(); return scanner; } @@ -230,9 +212,6 @@ public RegionScanner preScannerOpen( public RegionScanner postScannerOpen( ObserverContext e, Scan scan, RegionScanner scanner) throws IOException { - if (!isFlowRunRegion) { - return scanner; - } return new FlowScanner(e.getEnvironment(), scan, scanner, FlowScannerOperation.READ); } @@ -241,9 +220,6 @@ public RegionScanner postScannerOpen( public InternalScanner preFlush( ObserverContext c, Store store, InternalScanner scanner) throws IOException { - if (!isFlowRunRegion) { - return scanner; - } if (LOG.isDebugEnabled()) { if (store != null) { LOG.debug("preFlush store = " + store.getColumnFamilyName() @@ -264,9 +240,6 @@ public InternalScanner preFlush( @Override public void postFlush(ObserverContext c, Store store, StoreFile resultFile) { - if (!isFlowRunRegion) { - return; - } if (LOG.isDebugEnabled()) { if (store != null) { LOG.debug("postFlush store = " + store.getColumnFamilyName() @@ -288,9 +261,6 @@ public InternalScanner preCompact( InternalScanner scanner, ScanType scanType, CompactionRequest request) throws IOException { - if (!isFlowRunRegion) { - return scanner; - } FlowScannerOperation requestOp = FlowScannerOperation.MINOR_COMPACTION; if (request != null) { requestOp = (request.isMajor() ? FlowScannerOperation.MAJOR_COMPACTION diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java index 4cd581b..ec973cb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java @@ -23,13 +23,14 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Coprocessor; /** * The flow run table has column family info @@ -133,29 +134,17 @@ public void createTable(Admin admin, Configuration hbaseConf) infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS); // TODO: figure the split policy - flowRunTableDescp.addCoprocessor(FlowRunCoprocessor.class - .getCanonicalName()); + String coprocessorJarPathStr = hbaseConf.get( + YarnConfiguration.FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION, + YarnConfiguration.DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR); + + Path coprocessorJarPath = new Path(coprocessorJarPathStr); + LOG.info("CoprocessorJarPath=" + coprocessorJarPath.toString()); + flowRunTableDescp.addCoprocessor( + FlowRunCoprocessor.class.getCanonicalName(), coprocessorJarPath, + Coprocessor.PRIORITY_USER, null); admin.createTable(flowRunTableDescp); LOG.info("Status of table creation for " + table.getNameAsString() + "=" + admin.tableExists(table)); } - - public static boolean isFlowRunTable(HRegionInfo hRegionInfo, - Configuration conf) { - String regionTableName = hRegionInfo.getTable().getNameAsString(); - if (LOG.isDebugEnabled()) { - LOG.debug("regionTableName=" + regionTableName); - } - String flowRunTableName = BaseTable.getTableName(conf, - FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME) - .getNameAsString(); - if (flowRunTableName.equalsIgnoreCase(regionTableName)) { - if (LOG.isDebugEnabled()) { - LOG.debug(" table is the flow run table!! " - + flowRunTableName); - } - return true; - } - return false; - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md index 0d77f2c..6044202 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md @@ -216,30 +216,32 @@ http://hbase.apache.org/book.html#standalone.over.hdfs . Once you have an Apache HBase cluster ready to use, perform the following steps. ##### Step 2) Enable the coprocessor +In this version, the coprocessor is loaded dynamically (table coprocessor for the `flowrun` table). -Step 2.1) Add the timeline service jar to the HBase classpath in all HBase machines in the cluster. It -is needed for the coprocessor as well as the schema creator. For example, +Copy the timeline service jar to HDFS from where HBase can load it. It +is needed for the `flowrun` table creation in the schema creator. The default HDFS location is `/hbase/coprocessor`. +For example, - cp hadoop-yarn-server-timelineservice-3.0.0-alpha1-SNAPSHOT.jar /usr/hbase/lib/ + hadoop fs -mkdir /hbase/coprocessor + hadoop fs -put hadoop-yarn-server-timelineservice-3.0.0-alpha1-SNAPSHOT.jar + /hbase/coprocessor/hadoop-yarn-server-timelineservice.jar -Step 2.2) Enable the coprocessor that handles the aggregation. To enable it, add the following entry in -region servers' `hbase-site.xml` file (generally located in the `conf` directory) as follows: + +If you want to place the jar at a different location on hdfs, there also exists a yarn +configuration setting called `yarn.timeline-service.hbase.coprocessor.jar.hdfs.location`. +For example, ``` - hbase.coprocessor.region.classes - org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunCoprocessor + yarn.timeline-service.hbase.coprocessor.jar.hdfs.location + /custom/hdfs/path/jarName ``` -Step 2.3) Restart the region servers and the master to pick up the timeline service jar as well -as the config change. In this version, the coprocessor is loaded statically -(i.e. system coprocessor) as opposed to a dynamically (table coprocessor). - ##### Step 3) Create the timeline service schema Finally, run the schema creator tool to create the necessary tables: - bin/hbase org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator + bin/hadoop org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator The `TimelineSchemaCreator` tool supports a few options that may come handy especially when you are testing. For example, you can use `-skipExistingTable` (`-s` for short) to skip existing tables