From f3b66dcc47b42557d3e12546518384644ffe9ff1 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 28 Aug 2019 01:25:18 +0530 Subject: [PATCH] YARN-9789. Disable option for Write Ahead Logs. --- .../capacity/conf/LeveldbConfigurationStore.java | 12 ++++++----- .../capacity/conf/ZKConfigurationStore.java | 22 +++++++++++---------- .../capacity/conf/TestZKConfigurationStore.java | 23 ++++++++++++++++++++++ 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java index 4eb328c..d415db8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java @@ -185,12 +185,14 @@ public void close() throws IOException { @Override public void logMutation(LogMutation logMutation) throws IOException { - LinkedList logs = deserLogMutations(db.get(bytes(LOG_KEY))); - logs.add(logMutation); - if (logs.size() > maxLogs) { - logs.removeFirst(); + if (maxLogs > 0) { + LinkedList logs = deserLogMutations(db.get(bytes(LOG_KEY))); + logs.add(logMutation); + if (logs.size() > maxLogs) { + logs.removeFirst(); + } + db.put(bytes(LOG_KEY), serLogMutations(logs)); } - db.put(bytes(LOG_KEY), serLogMutations(logs)); pendingMutation = logMutation; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index 34c73ec..0a2b64a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -147,17 +147,19 @@ public synchronized void storeVersion() throws Exception { @Override public void logMutation(LogMutation logMutation) throws Exception { - byte[] storedLogs = zkManager.getData(logsPath); - LinkedList logs = new LinkedList<>(); - if (storedLogs != null) { - logs = (LinkedList) deserializeObject(storedLogs); - } - logs.add(logMutation); - if (logs.size() > maxLogs) { - logs.remove(logs.removeFirst()); + if (maxLogs > 0) { + byte[] storedLogs = zkManager.getData(logsPath); + LinkedList logs = new LinkedList<>(); + if (storedLogs != null) { + logs = (LinkedList) deserializeObject(storedLogs); + } + logs.add(logMutation); + if (logs.size() > maxLogs) { + logs.remove(logs.removeFirst()); + } + zkManager.safeSetData(logsPath, serializeObject(logs), -1, zkAcl, + fencingNodePath); } - zkManager.safeSetData(logsPath, serializeObject(logs), -1, zkAcl, - fencingNodePath); pendingMutation = logMutation; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java index c09ae28..a8112a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; +import org.apache.hadoop.util.curator.ZKCuratorManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.curator.framework.CuratorFramework; @@ -197,6 +198,28 @@ public void testMaxLogs() throws Exception { assertEquals("val3", logs.get(1).getUpdates().get("key3")); } + + @Test + public void testDisableAuditLogs() throws Exception { + conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 0); + confStore.initialize(conf, schedConf, rmContext); + String znodeParentPath = conf.get(YarnConfiguration. + RM_SCHEDCONF_STORE_ZK_PARENT_PATH, + YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH); + String logsPath = ZKCuratorManager.getNodePath(znodeParentPath, "LOGS"); + byte[] data = null; + ((ZKConfigurationStore) confStore).zkManager.setData(logsPath, data, -1); + + Map update = new HashMap<>(); + update.put("key1", "val1"); + YarnConfigurationStore.LogMutation mutation = + new YarnConfigurationStore.LogMutation(update, TEST_USER); + confStore.logMutation(mutation); + + data = ((ZKConfigurationStore) confStore).zkManager.getData(logsPath); + assertNull("Failed to Disable Audit Logs", data); + } + public Configuration createRMHAConf(String rmIds, String rmId, int adminPort) { Configuration conf = new YarnConfiguration(); -- 2.7.4 (Apple Git-66)