diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 5842d643579..16c989c5ca0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3659,6 +3659,12 @@ public static boolean areNodeLabelsEnabled( DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS = DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS; + public static final String + TIMELINE_SERVICE_READER_STORAGE_MONITOR_INTERVAL_MS = + TIMELINE_SERVICE_READER_PREFIX + "storage-monitor.interval-ms"; + public static final long + DEFAULT_TIMELINE_SERVICE_STORAGE_MONITOR_INTERVAL_MS = 60 * 1000; + /** * Marked collector properties as Private since it run as auxillary service. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java new file mode 100644 index 00000000000..9b8055b0983 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineReaderHBaseDown.java @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_SERVICE_READER_STORAGE_MONITOR_INTERVAL_MS;
+import static org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl.DATA_TO_RETRIEVE;
+import static org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl.MONITOR_FILTERS;
+
+public class TestTimelineReaderHBaseDown {
+
+ @Test(timeout=120000)
+ public void testTimelineReaderHBaseUp() throws Exception {
+ HBaseTestingUtility util = new HBaseTestingUtility();
+ configure(util);
+ try {
+ util.startMiniCluster();
+ DataGeneratorForTest.createSchema(util.getConfiguration());
+ DataGeneratorForTest.loadApps(util, System.currentTimeMillis());
+
+ TimelineReaderServer server = getTimelineReaderServer();
+ server.init(util.getConfiguration());
+ HBaseTimelineReaderImpl htr = getHBaseTimelineReaderImpl(server);
+ server.start();
+ checkQuery(htr);
+ } finally {
+ util.shutdownMiniCluster();
+ }
+ }
+
+ @Test(timeout=120000)
+ public void testTimelineReaderInitWhenHBaseIsDown() throws
+ TimeoutException, InterruptedException {
+ HBaseTestingUtility util = new HBaseTestingUtility();
+ configure(util);
+ TimelineReaderServer server = getTimelineReaderServer();
+
+ // init timeline reader when hbase is not running
+ server.init(util.getConfiguration());
+ HBaseTimelineReaderImpl htr = getHBaseTimelineReaderImpl(server);
+ server.start();
+ waitForHBaseDown(htr);
+ }
+
+ @Test(timeout=120000)
+ public void testTimelineReaderDetectsHBaseDown() throws Exception {
+ HBaseTestingUtility util = new HBaseTestingUtility();
+ configure(util);
+
+ try {
+ // start minicluster
+ util.startMiniCluster();
+ DataGeneratorForTest.createSchema(util.getConfiguration());
+ DataGeneratorForTest.loadApps(util, System.currentTimeMillis());
+
+ // init timeline reader
+ TimelineReaderServer server = getTimelineReaderServer();
+ server.init(util.getConfiguration());
+ HBaseTimelineReaderImpl htr = getHBaseTimelineReaderImpl(server);
+
+ // stop hbase after timeline reader init
+ util.shutdownMiniHBaseCluster();
+
+ // start server and check that it detects hbase is down
+ server.start();
+ waitForHBaseDown(htr);
+ } finally {
+ util.shutdownMiniCluster();
+ }
+ }
+
+ @Test(timeout=120000)
+ public void testTimelineReaderDetectsZooKeeperDown() throws Exception {
+ HBaseTestingUtility util = new HBaseTestingUtility();
+ configure(util);
+
+ try {
+ // start minicluster
+ util.startMiniCluster();
+ DataGeneratorForTest.createSchema(util.getConfiguration());
+ DataGeneratorForTest.loadApps(util, System.currentTimeMillis());
+
+ // init timeline reader
+ TimelineReaderServer server = getTimelineReaderServer();
+ server.init(util.getConfiguration());
+ HBaseTimelineReaderImpl htr = getHBaseTimelineReaderImpl(server);
+
+ // stop hbase and zookeeper after timeline reader init
+ util.shutdownMiniCluster();
+
+ // start server and check that it detects hbase is down
+ server.start();
+ waitForHBaseDown(htr);
+ } finally {
+ util.shutdownMiniCluster();
+ }
+ }
+
+ @Test(timeout=120000)
+ public void testTimelineReaderRecoversAfterHBaseReturns() throws Exception {
+ HBaseTestingUtility util = new HBaseTestingUtility();
+ configure(util);
+
+ try {
+ // start minicluster
+ util.startMiniCluster();
+ DataGeneratorForTest.createSchema(util.getConfiguration());
+ DataGeneratorForTest.loadApps(util, System.currentTimeMillis());
+
+ // init timeline reader
+ TimelineReaderServer server = getTimelineReaderServer();
+ server.init(util.getConfiguration());
+ HBaseTimelineReaderImpl htr = getHBaseTimelineReaderImpl(server);
+
+ // stop hbase after timeline reader init
+ util.shutdownMiniHBaseCluster();
+
+ // start server and check that it detects hbase is down
+ server.start();
+ waitForHBaseDown(htr);
+
+ util.startMiniHBaseCluster(1, 1);
+ GenericTestUtils.waitFor(() -> !htr.isHBaseDown(), 1000, 20000);
+ } finally {
+ util.shutdownMiniCluster();
+ }
+ }
+
+ private static void waitForHBaseDown(HBaseTimelineReaderImpl htr) throws
+ TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(() -> htr.isHBaseDown(), 1000, 20000);
+ try {
+ checkQuery(htr);
+ Assert.fail("Query should fail when HBase is down");
+ } catch (IOException e) {
+ Assert.assertEquals("HBase is down", e.getMessage());
+ }
+ }
+
+ private static void checkQuery(HBaseTimelineReaderImpl htr) throws
+ IOException {
+ TimelineReaderContext context =
+ new TimelineReaderContext(YarnConfiguration.DEFAULT_RM_CLUSTER_ID,
+ null, null, null, null, TimelineEntityType
+ .YARN_FLOW_ACTIVITY.toString(), null, null);
+ Set