From 6e0163e5e96490f063dccbcb7640e4fcfd519c28 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 22 Feb 2019 21:35:12 +0530 Subject: [PATCH] YARN-9317 --- .../yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 5 ++++- .../containermanager/ContainerManagerImpl.java | 15 +++++++++------ .../server/resourcemanager/ApplicationMasterService.java | 5 ++++- .../yarn/server/resourcemanager/ClientRMService.java | 5 ++++- .../yarn/server/resourcemanager/DefaultAMSProcessor.java | 6 ++++-- .../hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 ++++- .../server/resourcemanager/ResourceTrackerService.java | 9 +++++---- .../server/resourcemanager/amlauncher/AMLauncher.java | 5 ++++- 8 files changed, 38 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index b570625..08e7372 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -158,6 +158,7 @@ private NMNodeAttributesHandler nodeAttributesHandler; private NodeLabelsProvider nodeLabelsProvider; private NodeAttributesProvider nodeAttributesProvider; + private boolean timelineServiceV2Enabled; public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) { @@ -252,6 +253,8 @@ protected void serviceInit(Configuration conf) throws Exception { this.logAggregationEnabled = conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } @Override @@ -1406,7 +1409,7 @@ public void run() { newResource.toString()); } } - if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) { + if (timelineServiceV2Enabled) { updateTimelineCollectorData(response); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 01d70af..c827fcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -227,6 +227,7 @@ // NM metrics publisher is set only if the timeline service v.2 is enabled private NMTimelinePublisher nmMetricsPublisher; + private boolean timelineServiceV2Enabled; public ContainerManagerImpl(Context context, ContainerExecutor exec, DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater, @@ -263,11 +264,13 @@ public ContainerManagerImpl(Context context, ContainerExecutor exec, // initialize the metrics publisher if the timeline service v.2 is enabled // and the system publisher is enabled Configuration conf = context.getConf(); - if (YarnConfiguration.timelineServiceV2Enabled(conf) && - YarnConfiguration.systemMetricsPublisherEnabled(conf)) { - LOG.info("YARN system metrics publishing service is enabled"); - nmMetricsPublisher = createNMTimelinePublisher(context); - context.setNMTimelinePublisher(nmMetricsPublisher); + if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { + LOG.info("YARN system metrics publishing service is enabled"); + nmMetricsPublisher = createNMTimelinePublisher(context); + context.setNMTimelinePublisher(nmMetricsPublisher); + } + this.timelineServiceV2Enabled = true; } this.containersMonitor = createContainersMonitor(exec); addService(this.containersMonitor); @@ -1184,7 +1187,7 @@ protected void startContainerInternal( private FlowContext getFlowContext(ContainerLaunchContext launchContext, ApplicationId applicationID) { FlowContext flowContext = null; - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { String flowName = launchContext.getEnvironment() .get(TimelineUtils.FLOW_NAME_TAG_PREFIX); String flowVersion = launchContext.getEnvironment() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 70b7498..d9527d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -95,6 +95,7 @@ new ConcurrentHashMap(); protected final RMContext rmContext; private final AMSProcessingChain amsProcessingChain; + private boolean timelineServiceV2Enabled; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { @@ -212,6 +213,8 @@ protected void serviceStart() throws Exception { YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); super.serviceStart(); } @@ -302,7 +305,7 @@ public FinishApplicationMasterResponse finishApplicationMaster( rmContext.getRMApps().get(applicationAttemptId.getApplicationId()); // Remove collector address when app get finished. - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { ((RMAppImpl) rmApp).removeCollectorData(); } // checking whether the app exits in RMStateStore at first not to throw diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 062f39a..e2765a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -236,6 +236,7 @@ RMAppState.ACCEPTED, RMAppState.RUNNING); private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -306,6 +307,8 @@ protected void serviceStart() throws Exception { YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); super.serviceStart(); } @@ -585,7 +588,7 @@ public SubmitApplicationResponse submitApplication( throw RPCUtil.getRemoteException(ie); } - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { // Sanity check for flow run String value = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index b2c5ef3..9e05eda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -115,12 +115,15 @@ private RMContext rmContext; private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; @Override public void init(ApplicationMasterServiceContext amsContext, ApplicationMasterServiceProcessor nextProcessor) { this.rmContext = (RMContext)amsContext; this.resourceProfilesManager = rmContext.getResourceProfilesManager(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(rmContext.getYarnConfiguration()); } @Override @@ -326,8 +329,7 @@ public void allocate(ApplicationAttemptId appAttemptId, response.setNumClusterNodes(getScheduler().getNumClusterNodes()); // add collector address for this application - if (YarnConfiguration.timelineServiceV2Enabled( - getRmContext().getYarnConfiguration())) { + if (timelineServiceV2Enabled) { CollectorInfo collectorInfo = app.getCollectorInfo(); if (collectorInfo != null) { response.setCollectorInfo(collectorInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 3c5436c..0679b0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -95,6 +95,7 @@ private final ApplicationACLsManager applicationACLsManager; private Configuration conf; private YarnAuthorizationProvider authorizer; + private boolean timelineServiceV2Enabled; public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, @@ -115,6 +116,8 @@ public RMAppManager(RMContext context, this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } this.authorizer = YarnAuthorizationProvider.getInstance(conf); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } /** @@ -493,7 +496,7 @@ private RMAppImpl createAndPopulateNewRMApp( throw new YarnException(message); } - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { // Start timeline collector for the submitted app application.startTimelineCollector(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index f3d8eb2..9cff85b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -127,6 +127,7 @@ private DynamicResourceConfiguration drConf; private final AtomicLong timelineCollectorVersion = new AtomicLong(0); + private boolean timelineServiceV2Enabled; public ResourceTrackerService(RMContext rmContext, NodesListManager nodesListManager, @@ -173,6 +174,8 @@ protected void serviceInit(Configuration conf) throws Exception { minimumNodeManagerVersion = conf.get( YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); + timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); if (YarnConfiguration.areNodeLabelsEnabled(conf)) { isDistributedNodeLabelsConf = @@ -600,9 +603,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) NodeAction.SHUTDOWN, message); } - boolean timelineV2Enabled = - YarnConfiguration.timelineServiceV2Enabled(getConfig()); - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Check & update collectors info from request. updateAppCollectorsMap(request); } @@ -622,7 +623,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials); } - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Return collectors' map that NM needs to know setAppCollectorsMapToResponse(rmNode.getRunningApps(), nodeHeartBeatResponse); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 0bedb52..3ec329f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -81,6 +81,7 @@ private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; + private boolean timelineServiceV2Enabled; @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -93,6 +94,8 @@ public AMLauncher(RMContext rmContext, RMAppAttempt application, this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } private void connect() throws IOException { @@ -239,7 +242,7 @@ protected void setupTokens( } private void setFlowContext(ContainerLaunchContext container) { - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { Map environment = container.getEnvironment(); ApplicationId applicationId = application.getAppAttemptId().getApplicationId(); -- 2.7.4 (Apple Git-66)