From ba737df4f2eb50dc77ea713e8953a3df09019920 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Wed, 8 Nov 2017 16:44:46 +0530 Subject: [PATCH] tmp-patch --- .../server/resourcemanager/ResourceManager.java | 12 ++-- .../resourcemanager/recovery/ZKRMStateStore.java | 72 ++++++++++++---------- .../capacity/conf/ZKConfigurationStore.java | 5 +- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 07f5a769293..727bc521740 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -358,7 +358,7 @@ protected EmbeddedElector createEmbeddedElector() throws IOException { conf.getBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, YarnConfiguration.DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED); if (curatorEnabled) { - this.zkManager = getAndStartZKManager(conf); + this.zkManager = createAndStartZKManager(conf); elector = new CuratorBasedElectorService(this); } else { elector = new ActiveStandbyElectorBasedElectorService(this); @@ -372,11 +372,8 @@ protected EmbeddedElector createEmbeddedElector() throws IOException { * @return ZooKeeper Curator manager. * @throws IOException If it cannot create the manager. */ - public synchronized ZKCuratorManager getAndStartZKManager(Configuration + public ZKCuratorManager createAndStartZKManager(Configuration config) throws IOException { - if (this.zkManager != null) { - return zkManager; - } ZKCuratorManager manager = new ZKCuratorManager(config); // Get authentication @@ -396,7 +393,10 @@ public synchronized ZKCuratorManager getAndStartZKManager(Configuration } manager.start(authInfos); - this.zkManager = manager; + return manager; + } + + public ZKCuratorManager getZKManager() { return zkManager; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 5d3ca452067..e9e747481ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -233,7 +233,8 @@ STORE, UPDATE, REMOVE - }; + } + ; /** * Encapsulates znode path and corresponding split index for hierarchical @@ -316,8 +317,8 @@ public synchronized void initInternal(Configuration conf) if (appIdNodeSplitIndex < 0 || appIdNodeSplitIndex > 4) { LOG.info("Invalid value " + appIdNodeSplitIndex + " for config " + YarnConfiguration.ZK_APPID_NODE_SPLIT_INDEX + " specified. " + - "Resetting it to " + - YarnConfiguration.DEFAULT_ZK_APPID_NODE_SPLIT_INDEX); + "Resetting it to " + + YarnConfiguration.DEFAULT_ZK_APPID_NODE_SPLIT_INDEX); appIdNodeSplitIndex = YarnConfiguration.DEFAULT_ZK_APPID_NODE_SPLIT_INDEX; } @@ -358,7 +359,10 @@ public synchronized void initInternal(Configuration conf) amrmTokenSecretManagerRoot = getNodePath(zkRootNodePath, AMRMTOKEN_SECRET_MANAGER_ROOT); reservationRoot = getNodePath(zkRootNodePath, RESERVATION_SYSTEM_ROOT); - zkManager = resourceManager.getAndStartZKManager(conf); + zkManager = resourceManager.getZKManager(); + if (zkManager == null) { + zkManager = resourceManager.createAndStartZKManager(conf); + } delegationTokenNodeSplitIndex = conf.getInt(YarnConfiguration.ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX, YarnConfiguration.DEFAULT_ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX); @@ -562,10 +566,10 @@ private void loadAMRMTokenSecretManagerState(RMState rmState) } else { AMRMTokenSecretManagerStatePBImpl stateData = new AMRMTokenSecretManagerStatePBImpl( - AMRMTokenSecretManagerStateProto.parseFrom(data)); + AMRMTokenSecretManagerStateProto.parseFrom(data)); rmState.amrmTokenSecretManagerState = AMRMTokenSecretManagerState.newInstance( - stateData.getCurrentMasterKey(), stateData.getNextMasterKey()); + stateData.getCurrentMasterKey(), stateData.getNextMasterKey()); } } @@ -692,7 +696,7 @@ private void loadRMAppStateFromAppNode(RMState rmState, String appNodePath, if (!appId.equals( appState.getApplicationSubmissionContext().getApplicationId())) { throw new YarnRuntimeException("The node name is different from the " + - "application id"); + "application id"); } rmState.appState.put(appId, appState); loadApplicationAttemptState(appState, appNodePath); @@ -722,7 +726,7 @@ private synchronized void loadRMAppState(RMState rmState) throws Exception { getNodePath(parentNodePath, leafNodeName), appIdStr); } } - } else if (!childNodeName.equals(RM_APP_ROOT_HIERARCHIES)){ + } else if (!childNodeName.equals(RM_APP_ROOT_HIERARCHIES)) { LOG.debug("Unknown child node with name " + childNodeName + " under" + appRoot); } @@ -914,28 +918,30 @@ private void handleApplicationAttemptStateOp( + path); } switch (operation) { - case UPDATE: - if (exists(path)) { - zkManager.safeSetData(path, attemptStateData, -1, zkAcl, - fencingNodePath); - } else { - zkManager.safeCreate(path, attemptStateData, zkAcl, - CreateMode.PERSISTENT, zkAcl, fencingNodePath); - if (LOG.isDebugEnabled()) { - LOG.debug("Path " + path + " for " + appAttemptId + " didn't exist." + - " Created a new znode to update the application attempt state."); + case UPDATE: + if (exists(path)) { + zkManager.safeSetData(path, attemptStateData, -1, zkAcl, + fencingNodePath); + } else { + zkManager.safeCreate(path, attemptStateData, zkAcl, + CreateMode.PERSISTENT, zkAcl, fencingNodePath); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Path " + path + " for " + appAttemptId + " didn't exist." + + " Created a new znode to update the application attempt state."); + } } - } - break; - case STORE: - zkManager.safeCreate(path, attemptStateData, zkAcl, CreateMode.PERSISTENT, - zkAcl, fencingNodePath); - break; - case REMOVE: - zkManager.safeDelete(path, zkAcl, fencingNodePath); - break; - default: - break; + break; + case STORE: + zkManager + .safeCreate(path, attemptStateData, zkAcl, CreateMode.PERSISTENT, + zkAcl, fencingNodePath); + break; + case REMOVE: + zkManager.safeDelete(path, zkAcl, fencingNodePath); + break; + default: + break; } } @@ -1128,7 +1134,7 @@ protected synchronized void storeRMDTMasterKeyState( LOG.debug("Storing RMDelegationKey_" + delegationKey.getKeyId()); } ByteArrayOutputStream os = new ByteArrayOutputStream(); - try(DataOutputStream fsOut = new DataOutputStream(os)) { + try (DataOutputStream fsOut = new DataOutputStream(os)) { delegationKey.write(fsOut); zkManager.safeCreate(nodeCreatePath, os.toByteArray(), zkAcl, CreateMode.PERSISTENT, zkAcl, fencingNodePath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index a0bba8c440d..d89951cb19c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -81,7 +81,10 @@ public void initialize(Configuration config, Configuration schedConf, this.znodeParentPath = conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH, YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH); - this.zkManager = rmContext.getResourceManager().getAndStartZKManager(conf); + this.zkManager = rmContext.getResourceManager().getZKManager(); + if (this.zkManager == null) { + this.zkManager = rmContext.getResourceManager().createAndStartZKManager(conf); + } this.zkAcl = ZKCuratorManager.getZKAcls(conf); this.zkVersionPath = getNodePath(znodeParentPath, ZK_VERSION_PATH); -- 2.13.6 (Apple Git-96)