diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index fb9fb534fff..9ff2b90bc82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2689,6 +2689,13 @@ public static boolean isAclEnabled(Configuration conf) { SCM_CLEANER_PREFIX + "period-mins"; public static final int DEFAULT_SCM_CLEANER_PERIOD_MINS = 24 * 60; + public static final String TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS = + TIMELINE_SERVICE_PREFIX + "schema-creator.class"; + + public static final String DEFAULT_TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS = + "org.apache.hadoop.yarn.server.timelineservice.storage" + + ".HBaseTimelineSchemaCreator"; + /** * Initial delay before the first cleaner task is scheduled. Specified in * minutes. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java index cf6a8544241..476021a597e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java @@ -57,7 +57,7 @@ public static void createSchema(final Configuration conf) // the coprocessor class is loaded from classpath conf.set(YarnConfiguration.FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION, " "); // now create all tables - TimelineSchemaCreator.createAllTables(conf, false); + HBaseTimelineSchemaCreator.createAllTables(conf, false); } public static void loadApps(HBaseTestingUtility util, long ts) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HbaseTimelineSchemaCreator.java similarity index 97% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HbaseTimelineSchemaCreator.java index c9f7cecdf56..20234ddfc19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HbaseTimelineSchemaCreator.java @@ -55,13 +55,13 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable -public final class TimelineSchemaCreator { - private TimelineSchemaCreator() { +public final class HbaseTimelineSchemaCreator implements SchemaCreator { + private HbaseTimelineSchemaCreator() { } - final static String NAME = TimelineSchemaCreator.class.getSimpleName(); + final static String NAME = HbaseTimelineSchemaCreator.class.getSimpleName(); private static final Logger LOG = - LoggerFactory.getLogger(TimelineSchemaCreator.class); + LoggerFactory.getLogger(HbaseTimelineSchemaCreator.class); private static final String SKIP_EXISTING_TABLE_OPTION_SHORT = "s"; private static final String APP_METRICS_TTL_OPTION_SHORT = "ma"; private static final String SUB_APP_METRICS_TTL_OPTION_SHORT = "msa"; @@ -73,7 +73,7 @@ private TimelineSchemaCreator() { private static final String HELP_SHORT = "h"; private static final String CREATE_TABLES_SHORT = "c"; - public static void main(String[] args) throws Exception { + public void createTimelineSchema(String[] args) throws Exception { LOG.info("Starting the schema creation"); Configuration hbaseConf = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/SchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/SchemaCreator.java new file mode 100644 index 00000000000..d04f85d253b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/SchemaCreator.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.storage; + +/** + * This interface is for creating Timeline Schema. The backend for Timeline + * Service have to implement this. + */ +public interface SchemaCreator { + + void createTimelineSchema(String[] args) throws Exception; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java new file mode 100644 index 00000000000..20ae8bfe0f1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.storage; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This creates the timeline schema for storing application timeline + * information. Each backend has to implement the {@link SchemaCreator} for + * creating the schema in its backend and should be configured in yarn-site.xml. + */ +public class TimelineSchemaCreator extends Configured implements Tool { + private static final Logger LOG = + LoggerFactory.getLogger(TimelineSchemaCreator.class); + + public static void main(String[] args) { + try { + int status = ToolRunner.run(new YarnConfiguration(), + new TimelineSchemaCreator(), args); + System.exit(status); + } catch (Exception e) { + LOG.error("Error while creating Timeline Schema : ", e); + } + } + + @Override + public int run(String[] args) throws Exception { + Configuration conf = getConf(); + return createTimelineSchema(args, conf); + } + + @VisibleForTesting + int createTimelineSchema(String[] args, Configuration conf) throws Exception { + String schemaCreatorClassName = conf.get( + YarnConfiguration.TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS); + LOG.info("Using {} for creating Timeline Service Schema ", + schemaCreatorClassName); + try { + Class schemaCreatorClass = Class.forName(schemaCreatorClassName); + if (SchemaCreator.class.isAssignableFrom(schemaCreatorClass)) { + SchemaCreator schemaCreator = (SchemaCreator) ReflectionUtils + .newInstance(schemaCreatorClass, conf); + schemaCreator.createTimelineSchema(args); + return 0; + } else { + throw new YarnRuntimeException("Class: " + schemaCreatorClassName + + " not instance of " + SchemaCreator.class.getCanonicalName()); + } + } catch (ClassNotFoundException e) { + throw new YarnRuntimeException("Could not instantiate TimelineReader: " + + schemaCreatorClassName, e); + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DummyTimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DummyTimelineSchemaCreator.java new file mode 100644 index 00000000000..98304fc8155 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DummyTimelineSchemaCreator.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.storage; + +/** + * Dummy Implementation of {@link SchemaCreator} for test. + */ +public class DummyTimelineSchemaCreator implements SchemaCreator { + + @Override + public void createTimelineSchema(String[] args) { + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineSchemaCreator.java new file mode 100644 index 00000000000..16b6d995d4f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineSchemaCreator.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.storage; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Assert; +import org.junit.Test; + +/** + * Test cases for {@link TimelineSchemaCreator}. + */ +public class TestTimelineSchemaCreator { + + @Test + public void testTimelineSchemaCreation() throws Exception { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS, + "org.apache.hadoop.yarn.server.timelineservice.storage" + + ".DummyTimelineSchemaCreator"); + TimelineSchemaCreator timelineSchemaCreator = new TimelineSchemaCreator(); + Assert.assertEquals(0, timelineSchemaCreator + .createTimelineSchema(new String[]{}, conf)); + } +} \ No newline at end of file