commit 1dda340da507670c6b782924b9776852e283219c Author: Misha Dmitriev Date: Tue Oct 24 14:20:25 2017 -0700 YARN-7386. Duplicate Strings in various places in Yarn memory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index d722cc58dbbeaddd7ff80f300ecbea8943374003..a9f2ee3d5c65c16654e06da6b57942f06901479b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerRetryContext; @@ -392,7 +393,8 @@ private void initEnv() { this.environment = new HashMap(); for (StringStringMapProto c : list) { - this.environment.put(c.getKey(), c.getValue()); + this.environment.put(StringInterner.weakIntern(c.getKey()), + StringInterner.weakIntern(c.getValue())); } } @@ -402,7 +404,10 @@ public void setEnvironment(final Map env) { return; initEnv(); this.environment.clear(); - this.environment.putAll(env); + for (Map.Entry e : env.entrySet()) { + this.environment.put(StringInterner.weakIntern(e.getKey()), + StringInterner.weakIntern(e.getValue())); + } } private void addEnvToProto() { @@ -464,7 +469,7 @@ private void initApplicationACLs() { for (ApplicationACLMapProto aclProto : list) { this.applicationACLS.put(ProtoUtils.convertFromProtoFormat(aclProto - .getAccessType()), aclProto.getAcl()); + .getAccessType()), StringInterner.weakIntern(aclProto.getAcl())); } } @@ -513,7 +518,10 @@ public void setApplicationACLs( return; initApplicationACLs(); this.applicationACLS.clear(); - this.applicationACLS.putAll(appACLs); + for (Map.Entry e : appACLs.entrySet()) { + this.applicationACLS.put(e.getKey(), + StringInterner.weakIntern(e.getValue())); + } } public ContainerRetryContext getContainerRetryContext() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index b6e22d15c64d3821240eaff3fa96d105e81b6e9c..be8493886bb4d8e9977c8cd34c25607b80f84c37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -181,7 +181,7 @@ public void setNodeHttpAddress(String nodeHttpAddress) { builder.clearNodeHttpAddress(); return; } - builder.setNodeHttpAddress(nodeHttpAddress); + builder.setNodeHttpAddress(nodeHttpAddress.intern()); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index cfb8a74f59a476e89bd9b200913105380fac0ba0..ae5f6b487c3b8e739c8b326557ad8b0a7d3bf849 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -425,12 +426,12 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, this.systemClock = SystemClock.getInstance(); this.applicationId = applicationId; - this.name = name; + this.name = StringInterner.weakIntern(name); this.rmContext = rmContext; this.dispatcher = rmContext.getDispatcher(); this.handler = dispatcher.getEventHandler(); this.conf = config; - this.user = user; + this.user = StringInterner.weakIntern(user); this.queue = queue; this.submissionContext = submissionContext; this.scheduler = scheduler; @@ -441,7 +442,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, } else { this.startTime = startTime; } - this.applicationType = applicationType; + this.applicationType = StringInterner.weakIntern(applicationType); this.applicationTags = applicationTags; this.amReqs = amReqs; if (submissionContext.getPriority() != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 4997bc60f3e6984fa7343b91eb158b9899089dd0..d896d29ba5b9c4cf6e2a62f75e0de0c41adcdf71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -1645,7 +1646,7 @@ public void transition(RMAppAttemptImpl appAttempt, ClusterMetrics.getMetrics().addAMRegisterDelay(delay); RMAppAttemptRegistrationEvent registrationEvent = (RMAppAttemptRegistrationEvent) event; - appAttempt.host = registrationEvent.getHost(); + appAttempt.host = StringInterner.weakIntern(registrationEvent.getHost()); appAttempt.rpcPort = registrationEvent.getRpcport(); appAttempt.originalTrackingUrl = sanitizeTrackingUrl(registrationEvent.getTrackingurl());