From 2c860cace2e35e5d5de9c06b58b0fdac29dbbd60 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Thu, 21 Feb 2019 12:30:17 +0530 Subject: [PATCH] YARN-9317 --- .../yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 6 +++++- .../containermanager/ContainerManagerImpl.java | 15 +++++++++------ .../server/resourcemanager/ApplicationMasterService.java | 6 +++++- .../yarn/server/resourcemanager/ClientRMService.java | 6 +++++- .../yarn/server/resourcemanager/DefaultAMSProcessor.java | 6 ++++-- .../hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 ++++- .../server/resourcemanager/ResourceTrackerService.java | 9 +++++---- .../server/resourcemanager/amlauncher/AMLauncher.java | 5 ++++- 8 files changed, 41 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 5762ffa..1f13eb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -160,6 +160,7 @@ private NodeLabelsProvider nodeLabelsProvider; private NodeAttributesProvider nodeAttributesProvider; private long tokenSequenceNo; + private boolean timelineServiceV2Enabled; public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) { @@ -254,6 +255,9 @@ protected void serviceInit(Configuration conf) throws Exception { this.logAggregationEnabled = conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); + } @Override @@ -1411,7 +1415,7 @@ public void run() { newResource.toString()); } } - if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) { + if (timelineServiceV2Enabled) { updateTimelineCollectorData(response); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 2ca63ae..f3f5f8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -230,6 +230,7 @@ // NM metrics publisher is set only if the timeline service v.2 is enabled private NMTimelinePublisher nmMetricsPublisher; + private boolean timelineServiceV2Enabled; public ContainerManagerImpl(Context context, ContainerExecutor exec, DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater, @@ -267,11 +268,13 @@ public ContainerManagerImpl(Context context, ContainerExecutor exec, // initialize the metrics publisher if the timeline service v.2 is enabled // and the system publisher is enabled Configuration conf = context.getConf(); - if (YarnConfiguration.timelineServiceV2Enabled(conf) && - YarnConfiguration.systemMetricsPublisherEnabled(conf)) { - LOG.info("YARN system metrics publishing service is enabled"); - nmMetricsPublisher = createNMTimelinePublisher(context); - context.setNMTimelinePublisher(nmMetricsPublisher); + if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { + LOG.info("YARN system metrics publishing service is enabled"); + nmMetricsPublisher = createNMTimelinePublisher(context); + context.setNMTimelinePublisher(nmMetricsPublisher); + } + this.timelineServiceV2Enabled = true; } this.containersMonitor = createContainersMonitor(exec); addService(this.containersMonitor); @@ -1191,7 +1194,7 @@ protected void startContainerInternal( private FlowContext getFlowContext(ContainerLaunchContext launchContext, ApplicationId applicationID) { FlowContext flowContext = null; - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { String flowName = launchContext.getEnvironment() .get(TimelineUtils.FLOW_NAME_TAG_PREFIX); String flowVersion = launchContext.getEnvironment() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 70b7498..7bc4260 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -95,6 +95,7 @@ new ConcurrentHashMap(); protected final RMContext rmContext; private final AMSProcessingChain amsProcessingChain; + private boolean timelineServiceV2Enabled; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { @@ -212,6 +213,9 @@ protected void serviceStart() throws Exception { YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); + super.serviceStart(); } @@ -302,7 +306,7 @@ public FinishApplicationMasterResponse finishApplicationMaster( rmContext.getRMApps().get(applicationAttemptId.getApplicationId()); // Remove collector address when app get finished. - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { ((RMAppImpl) rmApp).removeCollectorData(); } // checking whether the app exits in RMStateStore at first not to throw diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 2da000c..757c7b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -236,6 +236,7 @@ RMAppState.ACCEPTED, RMAppState.RUNNING); private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -306,6 +307,9 @@ protected void serviceStart() throws Exception { YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); + super.serviceStart(); } @@ -585,7 +589,7 @@ public SubmitApplicationResponse submitApplication( throw RPCUtil.getRemoteException(ie); } - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { // Sanity check for flow run String value = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index b2c5ef3..9e05eda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -115,12 +115,15 @@ private RMContext rmContext; private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; @Override public void init(ApplicationMasterServiceContext amsContext, ApplicationMasterServiceProcessor nextProcessor) { this.rmContext = (RMContext)amsContext; this.resourceProfilesManager = rmContext.getResourceProfilesManager(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(rmContext.getYarnConfiguration()); } @Override @@ -326,8 +329,7 @@ public void allocate(ApplicationAttemptId appAttemptId, response.setNumClusterNodes(getScheduler().getNumClusterNodes()); // add collector address for this application - if (YarnConfiguration.timelineServiceV2Enabled( - getRmContext().getYarnConfiguration())) { + if (timelineServiceV2Enabled) { CollectorInfo collectorInfo = app.getCollectorInfo(); if (collectorInfo != null) { response.setCollectorInfo(collectorInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 3c5436c..0679b0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -95,6 +95,7 @@ private final ApplicationACLsManager applicationACLsManager; private Configuration conf; private YarnAuthorizationProvider authorizer; + private boolean timelineServiceV2Enabled; public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, @@ -115,6 +116,8 @@ public RMAppManager(RMContext context, this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } this.authorizer = YarnAuthorizationProvider.getInstance(conf); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } /** @@ -493,7 +496,7 @@ private RMAppImpl createAndPopulateNewRMApp( throw new YarnException(message); } - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { // Start timeline collector for the submitted app application.startTimelineCollector(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index c50950b..9365096 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -128,6 +128,7 @@ private final AtomicLong timelineCollectorVersion = new AtomicLong(0); private boolean checkIpHostnameInRegistration; + private boolean timelineServiceV2Enabled; public ResourceTrackerService(RMContext rmContext, NodesListManager nodesListManager, @@ -177,6 +178,8 @@ protected void serviceInit(Configuration conf) throws Exception { minimumNodeManagerVersion = conf.get( YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); + timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); if (YarnConfiguration.areNodeLabelsEnabled(conf)) { isDistributedNodeLabelsConf = @@ -621,9 +624,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) NodeAction.SHUTDOWN, message); } - boolean timelineV2Enabled = - YarnConfiguration.timelineServiceV2Enabled(getConfig()); - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Check & update collectors info from request. updateAppCollectorsMap(request); } @@ -639,7 +640,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) populateTokenSequenceNo(request, nodeHeartBeatResponse); - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Return collectors' map that NM needs to know setAppCollectorsMapToResponse(rmNode.getRunningApps(), nodeHeartBeatResponse); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index a734cb6..3ce014d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -84,6 +84,7 @@ private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; + private boolean timelineServiceV2Enabled; @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -96,6 +97,8 @@ public AMLauncher(RMContext rmContext, RMAppAttempt application, this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } private void connect() throws IOException { @@ -268,7 +271,7 @@ protected void setupTokens( } private void setFlowContext(ContainerLaunchContext container) { - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { Map environment = container.getEnvironment(); ApplicationId applicationId = application.getAppAttemptId().getApplicationId(); -- 2.7.4 (Apple Git-66)