From 0f109de0c11567ea37de1d705f195c280954bb7f Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Wed, 9 Aug 2017 19:07:52 +0530 Subject: [PATCH] YARN-6736 --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 5 + .../server/resourcemanager/ResourceManager.java | 24 +++-- .../metrics/TimelineServicePublishDelegator.java | 111 +++++++++++++++++++++ 3 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServicePublishDelegator.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b8cc4fd..54af876 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2402,6 +2402,11 @@ public static boolean isAclEnabled(Configuration conf) { public static final String PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT = "jdbc:phoenix:localhost:2181:/hbase"; + public static final String TIMELINE_SERVICE_UPGRADE_MODE = + TIMELINE_SERVICE_PREFIX + "upgrade-mode"; + + public static final boolean DEFAULT_TIMELINE_SERVICE_UPGRADE_MODE = false; + // /////////////////////////////// // Shared Cache Configs // /////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 6606f0c..012f5a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; +import org.apache.hadoop.yarn.server.resourcemanager.metrics.TimelineServicePublishDelegator; import org.apache.hadoop.yarn.server.resourcemanager.metrics.TimelineServiceV1Publisher; import org.apache.hadoop.yarn.server.resourcemanager.metrics.TimelineServiceV2Publisher; import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy; @@ -328,11 +329,16 @@ protected void serviceInit(Configuration conf) throws Exception { // initialize the RM timeline collector first so that the system metrics // publisher can bind to it - if (YarnConfiguration.timelineServiceV2Enabled(this.conf)) { - RMTimelineCollectorManager timelineCollectorManager = - createRMTimelineCollectorManager(); - addService(timelineCollectorManager); - rmContext.setRMTimelineCollectorManager(timelineCollectorManager); + if (YarnConfiguration.timelineServiceEnabled(conf) + && YarnConfiguration.systemMetricsPublisherEnabled(conf)) { + if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_UPGRADE_MODE, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_UPGRADE_MODE) + || YarnConfiguration.timelineServiceV2Enabled(this.conf)) { + RMTimelineCollectorManager timelineCollectorManager = + createRMTimelineCollectorManager(); + addService(timelineCollectorManager); + rmContext.setRMTimelineCollectorManager(timelineCollectorManager); + } } SystemMetricsPublisher systemMetricsPublisher = @@ -509,7 +515,13 @@ protected SystemMetricsPublisher createSystemMetricsPublisher() { SystemMetricsPublisher publisher; if (YarnConfiguration.timelineServiceEnabled(conf) && YarnConfiguration.systemMetricsPublisherEnabled(conf)) { - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_UPGRADE_MODE, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_UPGRADE_MODE)) { + LOG.info( + "system metrics publisher with the timeline service V1 and V2 is " + + "configured"); + publisher = new TimelineServicePublishDelegator(this); + } else if (YarnConfiguration.timelineServiceV2Enabled(conf)) { // we're dealing with the v.2.x publisher LOG.info("system metrics publisher with the timeline service V2 is " + "configured"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServicePublishDelegator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServicePublishDelegator.java new file mode 100644 index 0000000..b5ea0f2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServicePublishDelegator.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.metrics; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; + +public class TimelineServicePublishDelegator extends CompositeService + implements SystemMetricsPublisher { + ResourceManager rm; + + TimelineServiceV1Publisher v1Publisher; + TimelineServiceV2Publisher v2Publisher; + + public TimelineServicePublishDelegator(ResourceManager rm) { + super("TimelineServicePublishDelegator"); + this.rm = rm; + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + v1Publisher = new TimelineServiceV1Publisher(); + addIfService(v1Publisher); + v2Publisher = new TimelineServiceV2Publisher( + this.rm.getRMContext().getRMTimelineCollectorManager()); + addIfService(v2Publisher); + super.serviceInit(conf); + } + + @Override + public void appCreated(RMApp app, long createdTime) { + v1Publisher.appCreated(app, createdTime); + v2Publisher.appCreated(app, createdTime); + } + + @Override + public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) { + v1Publisher.appACLsUpdated(app, appViewACLs, updatedTime); + v2Publisher.appACLsUpdated(app, appViewACLs, updatedTime); + } + + @Override + public void appUpdated(RMApp app, long updatedTime) { + v1Publisher.appUpdated(app, updatedTime); + v2Publisher.appUpdated(app, updatedTime); + } + + @Override + public void appStateUpdated(RMApp app, YarnApplicationState appState, + long updatedTime) { + v1Publisher.appStateUpdated(app, appState, updatedTime); + v2Publisher.appStateUpdated(app, appState, updatedTime); + } + + @Override + public void appFinished(RMApp app, RMAppState state, long finishedTime) { + v1Publisher.appFinished(app, state, finishedTime); + v2Publisher.appFinished(app, state, finishedTime); + } + + @Override + public void appAttemptRegistered(RMAppAttempt appAttempt, + long registeredTime) { + v1Publisher.appAttemptRegistered(appAttempt, registeredTime); + v2Publisher.appAttemptRegistered(appAttempt, registeredTime); + } + + @Override + public void appAttemptFinished(RMAppAttempt appAttempt, + RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) { + v1Publisher.appAttemptFinished(appAttempt, appAttemtpState, app, + finishedTime); + v2Publisher.appAttemptFinished(appAttempt, appAttemtpState, app, + finishedTime); + } + + @Override + public void containerCreated(RMContainer container, long createdTime) { + v1Publisher.containerCreated(container, createdTime); + v2Publisher.containerCreated(container, createdTime); + } + + @Override + public void containerFinished(RMContainer container, long finishedTime) { + v1Publisher.containerFinished(container, finishedTime); + v2Publisher.containerFinished(container, finishedTime); + } +} -- 2.10.1 (Apple Git-78)