diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7f9c21a..56b0f65 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1910,6 +1910,15 @@ public static boolean isAclEnabled(Configuration conf) {
+ "hbase.coprocessor.app-final-value-retention-milliseconds";
/**
+ * The name for setting that points to an optional HBase configuration
+ * (hbase-site.xml file) with settings that will override the ones found on
+ * the classpath. .
+ */
+ public static final String TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE =
+ TIMELINE_SERVICE_PREFIX
+ + "hbase.configuration.file";
+
+ /**
* The setting that controls how long the final value of a metric of a
* completed app is retained before merging into the flow sum. Up to this time
* after an application is completed out-of-order values that arrive can be
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 5a94f69..ad044fa 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2227,6 +2227,17 @@
259200000
+
+ Optional URL to an hbase-site.xml configuration file to be
+ used to connect to the timeline-service hbase cluster. If empty or not
+ specified, then the HBase configuration will be loaded from the classpath.
+ When specified the values in the specified configuration file will override
+ those from the ones that are present on the classpath.
+
+ yarn.timeline-service.hbase.configuration.file
+
+
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineStorageUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineStorageUtils.java
new file mode 100644
index 0000000..7f537b0
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineStorageUtils.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class TestTimelineStorageUtils {
+ private static HBaseTestingUtility hbaseTestUtil;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ hbaseTestUtil = new HBaseTestingUtility();
+ }
+
+ @Rule
+ public final TemporaryFolder folder = new TemporaryFolder();
+
+ @Test
+ public void testInvertInt() {
+ assertEquals(Integer.MIN_VALUE,
+ TimelineStorageUtils.invertInt(TimelineStorageUtils
+ .invertInt(Integer.MIN_VALUE)));
+ assertEquals(-128,
+ TimelineStorageUtils.invertInt(TimelineStorageUtils.invertInt(-128)));
+ assertEquals(-127,
+ TimelineStorageUtils.invertInt(TimelineStorageUtils.invertInt(-127)));
+ assertEquals(-1,
+ TimelineStorageUtils.invertInt(TimelineStorageUtils.invertInt(-1)));
+ assertEquals(0,
+ TimelineStorageUtils.invertInt(TimelineStorageUtils.invertInt(0)));
+ assertEquals(1,
+ TimelineStorageUtils.invertInt(TimelineStorageUtils.invertInt(1)));
+ assertEquals(127,
+ TimelineStorageUtils.invertInt(TimelineStorageUtils.invertInt(127)));
+ assertEquals(128,
+ TimelineStorageUtils.invertInt(TimelineStorageUtils.invertInt(128)));
+ assertEquals(Integer.MAX_VALUE,
+ TimelineStorageUtils.invertInt(TimelineStorageUtils
+ .invertInt(Integer.MAX_VALUE)));
+ }
+
+ @Test
+ public void testGetTimelineServiceHBaseConf() throws IOException {
+ int megabyte = 1024 * 1024;
+ String hbaseClientProperty =
+ "hbase.dfs.client.read.shortcircuit.buffer.size";
+ Configuration testHBaseConf = hbaseTestUtil.getConfiguration();
+ testHBaseConf.setInt(hbaseClientProperty, megabyte);
+
+ assertEquals(megabyte, testHBaseConf.getInt(hbaseClientProperty, 0));
+
+ Configuration timelineHBaseConf =
+ TimelineStorageUtils.getTimelineServiceHBaseConf(testHBaseConf);
+
+ timelineHBaseConf.setInt(hbaseClientProperty, 2 * megabyte);
+ // Confirm that the original conf hasn't changed
+ assertEquals(megabyte, testHBaseConf.getInt(hbaseClientProperty, 0));
+ // And confirm that the conf for HBase _has_ changed.
+ assertEquals(2 * megabyte, timelineHBaseConf.getInt(hbaseClientProperty, 0));
+
+ // We expect TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE not to be set by
+ // default
+ String timelineServiceHBaseConfFile =
+ testHBaseConf.get(
+ YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE, null);
+ assertNull(timelineServiceHBaseConfFile);
+
+ // bump the setting up to 3MB to be able to distinguish
+ timelineHBaseConf.setInt(hbaseClientProperty, 3 * megabyte);
+ assertEquals(3 * megabyte, timelineHBaseConf.getInt(hbaseClientProperty, 0));
+
+ // Create a new file
+ File hbaseSiteXMLFile = folder.newFile("hbase-site.xml");
+
+ FileOutputStream fos = new FileOutputStream(hbaseSiteXMLFile);
+ timelineHBaseConf.writeXml(fos);
+
+ // The hbase-site.xml in the tmp directory should have a setting
+ // of 3MB
+
+ String hBaseXMLFileURL = hbaseSiteXMLFile.toURI().toURL().toString();
+
+ // Create a clean new conf
+ Configuration newTestHBaseConf = hbaseTestUtil.getConfiguration();
+ newTestHBaseConf.set(
+ YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE,
+ hBaseXMLFileURL);
+ newTestHBaseConf.setInt(hbaseClientProperty, 4 * megabyte);
+
+ Configuration newTimelineHBaseConf =
+ TimelineStorageUtils.getTimelineServiceHBaseConf(testHBaseConf);
+
+ // original conf should not be modified
+ assertEquals(4 * megabyte, newTestHBaseConf.getInt(hbaseClientProperty, 0));
+
+ // New HBase conf should have value read from hbase-site.xml
+ assertEquals(3 * megabyte,
+ newTimelineHBaseConf.getInt(hbaseClientProperty, 0));
+
+ }
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index a384a84..7894456 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -24,7 +24,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.service.AbstractService;
@@ -32,6 +31,7 @@
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader;
import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReaderFactory;
@@ -54,7 +54,8 @@ public HBaseTimelineReaderImpl() {
@Override
public void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
- hbaseConf = HBaseConfiguration.create(conf);
+ hbaseConf = TimelineStorageUtils.getTimelineServiceHBaseConf(conf);
+
conn = ConnectionFactory.createConnection(hbaseConf);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 3511a2f..16d59a4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -26,7 +26,6 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.service.AbstractService;
@@ -50,6 +49,7 @@
import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongKeyConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
@@ -112,7 +112,9 @@ public HBaseTimelineWriterImpl(Configuration conf) throws IOException {
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
- Configuration hbaseConf = HBaseConfiguration.create(conf);
+
+ Configuration hbaseConf =
+ TimelineStorageUtils.getTimelineServiceHBaseConf(conf);
conn = ConnectionFactory.createConnection(hbaseConf);
entityTable = new EntityTable().getTableMutator(hbaseConf, conn);
appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index 33f5449..aa0df54 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -34,13 +34,13 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
@@ -68,7 +68,8 @@ private TimelineSchemaCreator() {
public static void main(String[] args) throws Exception {
- Configuration hbaseConf = HBaseConfiguration.create();
+ Configuration hbaseConf =
+ TimelineStorageUtils.getTimelineServiceHBaseConf(new Configuration());
// Grab input args and allow for -Dxyz style arguments
String[] otherArgs = new GenericOptionsParser(hbaseConf, args)
.getRemainingArgs();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
index aa9a793..a07b767 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.common;
import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -32,6 +34,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Tag;
@@ -39,6 +42,7 @@
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
@@ -583,4 +587,33 @@ public static boolean isFlowRunTable(HRegionInfo hRegionInfo,
}
return false;
}
+
+ /**
+ * @param conf Yarn configuration
+ * @return a configuration with the HBase configuration from the classpath,
+ * optionally overwritten by the timeline service configuration URL if
+ * specified.
+ * @throws MalformedURLException if a timeline service HBase configuration URL
+ * is specified but is a malformed URL.
+ */
+ public static Configuration getTimelineServiceHBaseConf(Configuration conf)
+ throws MalformedURLException {
+ Configuration hbaseConf;
+
+ String timelineServiceHBaseConfFileURL =
+ conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE);
+ if (timelineServiceHBaseConfFileURL != null
+ && timelineServiceHBaseConfFileURL.length() > 0) {
+ // create a clone so that we don't mess with out input one
+ hbaseConf = new Configuration(conf);
+ Configuration plainHBaseConf = new Configuration(false);
+ URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL);
+ plainHBaseConf.addResource(hbaseSiteXML);
+ HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
+ } else {
+ // default to what is on the classpath
+ hbaseConf = HBaseConfiguration.create(conf);
+ }
+ return hbaseConf;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
index 6e151c9..4600d76 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
@@ -148,6 +148,7 @@ New configuration parameters that are introduced with v.2 are marked bold.
| `yarn.timeline-service.webapp.address` | The http address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8188`. |
| `yarn.timeline-service.webapp.https.address` | The https address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8190`. |
| **`yarn.timeline-service.writer.flush-interval-seconds`** | The setting that controls how often the timeline collector flushes the timeline writer. Defaults to `60`. |
+| **`yarn.timeline-service.hbase.configuration.file`** | Optional URL to an hbase-site.xml configuration file to be used to connect to the timeline-service hbase cluster. If empty or not specified, then the HBase configuration will be loaded from the classpath. When specified the values in the specified configuration file will override those from the ones that are present on the classpath. Defaults to `null`. |
| **`yarn.timeline-service.app-collector.linger-period.ms`** | Time period till which the application collector will be alive in NM, after the application master container finishes. Defaults to `1000` (1 second). |
| **`yarn.timeline-service.timeline-client.number-of-async-entities-to-merge`** | Time line V2 client tries to merge these many number of async entities (if available) and then call the REST ATS V2 API to submit. Defaults to `10`. |
| **`yarn.timeline-service.coprocessor.app-final-value-retention-milliseconds`** | The setting that controls how long the final value of a metric of a completed app is retained before merging into the flow sum. Defaults to `259200000` (3 days). |
@@ -212,6 +213,17 @@ Following are the basic configurations to start Timeline service v.2:
+ Optional URL to an hbase-site.xml configuration file to be
+ used to connect to the timeline-service hbase cluster. If empty or not
+ specified, then the HBase configuration will be loaded from the classpath.
+ When specified the values in the specified configuration file will override
+ those from the ones that are present on the classpath.
+
+ yarn.timeline-service.hbase.configuration.file
+ file:/etc/hbase/hbase-ats-dc1/hbase-site.xml
+
+
+
yarn.nodemanager.aux-services
mapreduce_shuffle,timeline_collector