diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9a1eb54..02c2338 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -464,6 +464,9 @@ private static void addDeprecatedKeys() { public static final String ZK_RM_STATE_STORE_ROOT_NODE_ACL = ZK_STATE_STORE_PREFIX + "root-node.acl"; + /** Maximum object size default for znodes */ + public static final long JUTE_MAXBUFFER_DEFAULT = 1048575; + /** HA related configs */ public static final String RM_HA_PREFIX = RM_PREFIX + "ha."; public static final String RM_HA_ENABLED = RM_HA_PREFIX + "enabled"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index ca0f4ac..f11209d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto; @@ -140,6 +141,7 @@ private static final String RM_DT_MASTER_KEYS_ROOT_ZNODE_NAME = "RMDTMasterKeysRoot"; + private Long juteMaxbuffer = getJuteMaxbuffer(); private String zkHostPort = null; private int numRetries; private int zkSessionTimeout; @@ -326,6 +328,11 @@ private void setRootNodeAcls() throws Exception { } } + private Long getJuteMaxbuffer() { + return Long.getLong(System.getProperty("jute.maxbuffer"), + YarnConfiguration.JUTE_MAXBUFFER_DEFAULT); + } + @Override protected synchronized void closeInternal() throws Exception { if (verifyActiveStatusThread != null) { @@ -589,9 +596,13 @@ public synchronized void storeApplicationStateInternal(ApplicationId appId, LOG.debug("Storing info for app: " + appId + " at: " + nodeCreatePath); } byte[] appStateData = appStateDataPB.getProto().toByteArray(); + if (appStateData.length > juteMaxbuffer) { + throw new YarnException("jute.maxBuffer exceeded for app: " + appId + + "appStateData.length = " + appStateData.length + + " jute.maxbuffer = " + juteMaxbuffer); + } safeCreate(nodeCreatePath, appStateData, zkAcl, CreateMode.PERSISTENT); - } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 66b023c..ea04d27 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl; @@ -105,6 +106,7 @@ public void cleanupCuratorServer() throws IOException { public TestZKRMStateStoreInternal(Configuration conf, String workingZnode) throws Exception { + conf.setInt(YarnConfiguration.RM_ZK_NUM_RETRIES,5); init(conf); start(); assertTrue(znodeWorkingPath.equals(workingZnode)); @@ -501,4 +503,17 @@ public void testDuplicateRMAppDeletion() throws Exception { } store.close(); } + + @Test(expected = YarnException.class) + public void testAppStateSize() throws Exception { + TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester(); + RMStateStore store = zkTester.getRMStateStore(); + long submitTime = System.currentTimeMillis(); + ApplicationId appId = ApplicationId.newInstance(submitTime, 1234); + ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); + byte [] userbytes = new byte[10000000]; + String username = new String(userbytes, "UTF-8"); + ApplicationStateData appState = ApplicationStateData.newInstance(submitTime,submitTime,context,username); + store.storeApplicationStateInternal(appId, appState); + } }