From b31842a4bf66be286ec2fe4b4d3341201cd25a61 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 22 Feb 2019 21:56:48 +0530 Subject: [PATCH] YARN-9317 --- .../yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 5 ++++- .../containermanager/ContainerManagerImpl.java | 15 +++++++++------ .../server/resourcemanager/ApplicationMasterService.java | 5 ++++- .../yarn/server/resourcemanager/ClientRMService.java | 5 ++++- .../yarn/server/resourcemanager/DefaultAMSProcessor.java | 6 ++++-- .../hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 ++++- .../server/resourcemanager/ResourceTrackerService.java | 9 +++++---- .../server/resourcemanager/amlauncher/AMLauncher.java | 5 ++++- 8 files changed, 38 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index e79ab47..8eb89dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -153,6 +153,7 @@ private NMNodeLabelsHandler nodeLabelsHandler; private final NodeLabelsProvider nodeLabelsProvider; + private boolean timelineServiceV2Enabled; public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) { @@ -240,6 +241,8 @@ protected void serviceInit(Configuration conf) throws Exception { this.logAggregationEnabled = conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } @Override @@ -1177,7 +1180,7 @@ public void run() { newResource.toString()); } } - if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) { + if (timelineServiceV2Enabled) { updateTimelineCollectorData(response); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 41b7418..3c63ec6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -227,6 +227,7 @@ // NM metrics publisher is set only if the timeline service v.2 is enabled private NMTimelinePublisher nmMetricsPublisher; + private boolean timelineServiceV2Enabled; public ContainerManagerImpl(Context context, ContainerExecutor exec, DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater, @@ -263,11 +264,13 @@ public ContainerManagerImpl(Context context, ContainerExecutor exec, // initialize the metrics publisher if the timeline service v.2 is enabled // and the system publisher is enabled Configuration conf = context.getConf(); - if (YarnConfiguration.timelineServiceV2Enabled(conf) && - YarnConfiguration.systemMetricsPublisherEnabled(conf)) { - LOG.info("YARN system metrics publishing service is enabled"); - nmMetricsPublisher = createNMTimelinePublisher(context); - context.setNMTimelinePublisher(nmMetricsPublisher); + if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { + LOG.info("YARN system metrics publishing service is enabled"); + nmMetricsPublisher = createNMTimelinePublisher(context); + context.setNMTimelinePublisher(nmMetricsPublisher); + } + this.timelineServiceV2Enabled = true; } this.containersMonitor = createContainersMonitor(exec); addService(this.containersMonitor); @@ -1184,7 +1187,7 @@ protected void startContainerInternal( private FlowContext getFlowContext(ContainerLaunchContext launchContext, ApplicationId applicationID) { FlowContext flowContext = null; - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { String flowName = launchContext.getEnvironment() .get(TimelineUtils.FLOW_NAME_TAG_PREFIX); String flowVersion = launchContext.getEnvironment() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index ae28879..49bc21c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -96,6 +96,7 @@ new ConcurrentHashMap(); protected final RMContext rmContext; private final AMSProcessingChain amsProcessingChain; + private boolean timelineServiceV2Enabled; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { @@ -213,6 +214,8 @@ protected void serviceStart() throws Exception { YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); super.serviceStart(); } @@ -303,7 +306,7 @@ public FinishApplicationMasterResponse finishApplicationMaster( rmContext.getRMApps().get(applicationAttemptId.getApplicationId()); // Remove collector address when app get finished. - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { ((RMAppImpl) rmApp).removeCollectorData(); } // checking whether the app exits in RMStateStore at first not to throw diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 2ae1031..379251e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -222,6 +222,7 @@ RMAppState.ACCEPTED, RMAppState.RUNNING); private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -292,6 +293,8 @@ protected void serviceStart() throws Exception { YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); super.serviceStart(); } @@ -571,7 +574,7 @@ public SubmitApplicationResponse submitApplication( throw RPCUtil.getRemoteException(ie); } - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { // Sanity check for flow run String value = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 5168bd3..d3ce241 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -107,12 +107,15 @@ private RMContext rmContext; private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; @Override public void init(ApplicationMasterServiceContext amsContext, ApplicationMasterServiceProcessor nextProcessor) { this.rmContext = (RMContext)amsContext; this.resourceProfilesManager = rmContext.getResourceProfilesManager(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(rmContext.getYarnConfiguration()); } @Override @@ -318,8 +321,7 @@ public void allocate(ApplicationAttemptId appAttemptId, response.setNumClusterNodes(getScheduler().getNumClusterNodes()); // add collector address for this application - if (YarnConfiguration.timelineServiceV2Enabled( - getRmContext().getYarnConfiguration())) { + if (timelineServiceV2Enabled) { CollectorInfo collectorInfo = app.getCollectorInfo(); if (collectorInfo != null) { response.setCollectorInfo(collectorInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 9369eb2..7f9f51c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -94,6 +94,7 @@ private final ApplicationACLsManager applicationACLsManager; private Configuration conf; private YarnAuthorizationProvider authorizer; + private boolean timelineServiceV2Enabled; public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, @@ -114,6 +115,8 @@ public RMAppManager(RMContext context, this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } this.authorizer = YarnAuthorizationProvider.getInstance(conf); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } /** @@ -451,7 +454,7 @@ private RMAppImpl createAndPopulateNewRMApp( throw new YarnException(message); } - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { // Start timeline collector for the submitted app application.startTimelineCollector(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index e997192..81be628 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -124,6 +124,7 @@ private DynamicResourceConfiguration drConf; private final AtomicLong timelineCollectorVersion = new AtomicLong(0); + private boolean timelineServiceV2Enabled; public ResourceTrackerService(RMContext rmContext, NodesListManager nodesListManager, @@ -170,6 +171,8 @@ protected void serviceInit(Configuration conf) throws Exception { minimumNodeManagerVersion = conf.get( YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); + timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); if (YarnConfiguration.areNodeLabelsEnabled(conf)) { isDistributedNodeLabelsConf = @@ -578,9 +581,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) NodeAction.SHUTDOWN, message); } - boolean timelineV2Enabled = - YarnConfiguration.timelineServiceV2Enabled(getConfig()); - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Check & update collectors info from request. updateAppCollectorsMap(request); } @@ -600,7 +601,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials); } - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Return collectors' map that NM needs to know setAppCollectorsMapToResponse(rmNode.getRunningApps(), nodeHeartBeatResponse); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 0bedb52..3ec329f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -81,6 +81,7 @@ private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; + private boolean timelineServiceV2Enabled; @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -93,6 +94,8 @@ public AMLauncher(RMContext rmContext, RMAppAttempt application, this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } private void connect() throws IOException { @@ -239,7 +242,7 @@ protected void setupTokens( } private void setFlowContext(ContainerLaunchContext container) { - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { Map environment = container.getEnvironment(); ApplicationId applicationId = application.getAppAttemptId().getApplicationId(); -- 2.7.4 (Apple Git-66)