diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
index 394454f..6d39366 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
@@ -578,7 +578,7 @@ SignalContainerResponse signalToContainer(
* Note: If application timeout value is less than or equal to current
* time then update application throws YarnException.
* @param request to set ApplicationTimeouts of an application
- * @return an empty response that the update has completed successfully.
+ * @return a response with updated timeouts.
* @throws YarnException if update request has empty values or application is
* in completing states.
* @throws IOException on IO failures
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UpdateApplicationTimeoutsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UpdateApplicationTimeoutsResponse.java
index bd02bb8..3770eb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UpdateApplicationTimeoutsResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UpdateApplicationTimeoutsResponse.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.util.Records;
/**
@@ -43,4 +44,22 @@ public static UpdateApplicationTimeoutsResponse newInstance() {
Records.newRecord(UpdateApplicationTimeoutsResponse.class);
return response;
}
+
+ /**
+ * Get ApplicationTimeouts of the application. Timeout value is
+ * in ISO8601 standard with format yyyy-MM-dd'T'HH:mm:ss.SSSZ.
+ * @return all ApplicationTimeouts of the application.
+ */
+ public abstract Map getApplicationTimeouts();
+
+ /**
+ * Set the ApplicationTimeouts for the application. Timeout value
+ * is absolute. Timeout value should meet ISO8601 format. Support ISO8601
+ * format is yyyy-MM-dd'T'HH:mm:ss.SSSZ. All pre-existing Map entries
+ * are cleared before adding the new Map.
+ * @param applicationTimeouts ApplicationTimeoutss for the
+ * application
+ */
+ public abstract void setApplicationTimeouts(
+ Map applicationTimeouts);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 893348a..5f6b300 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -41,6 +41,7 @@
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -359,10 +360,21 @@ private void updateApplicationTimeout(String applicationId,
+ timeoutType.toString() + " of an application " + applicationId);
UpdateApplicationTimeoutsRequest request = UpdateApplicationTimeoutsRequest
.newInstance(appId, Collections.singletonMap(timeoutType, newTimeout));
- client.updateApplicationTimeouts(request);
+ UpdateApplicationTimeoutsResponse updateApplicationTimeouts =
+ client.updateApplicationTimeouts(request);
+ String updatedTimeout =
+ updateApplicationTimeouts.getApplicationTimeouts().get(timeoutType);
+
+ if (timeoutType.equals(ApplicationTimeoutType.LIFETIME)
+ && !newTimeout.equals(updatedTimeout)) {
+ sysout.println("Updated lifetime of an application " + applicationId
+ + " to queue max/default lifetime." + " New expiry time is "
+ + updatedTimeout);
+ return;
+ }
sysout.println(
"Successfully updated " + timeoutType.toString() + " of an application "
- + applicationId + ". New expiry time is " + newTimeout);
+ + applicationId + ". New expiry time is " + updatedTimeout);
}
/**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 3c35b9c..13730f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -48,6 +48,7 @@
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -2148,17 +2149,16 @@ public void testUpdateApplicationTimeout() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 6);
- ApplicationReport appReport = ApplicationReport.newInstance(applicationId,
- ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue",
- "appname", "host", 124, null, YarnApplicationState.RUNNING,
- "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null,
- "N/A", 0.53789f, "YARN", null);
- ApplicationTimeout timeout = ApplicationTimeout
- .newInstance(ApplicationTimeoutType.LIFETIME, "N/A", -1);
- appReport.setApplicationTimeouts(
- Collections.singletonMap(timeout.getTimeoutType(), timeout));
- when(client.getApplicationReport(any(ApplicationId.class)))
- .thenReturn(appReport);
+ UpdateApplicationTimeoutsResponse response =
+ mock(UpdateApplicationTimeoutsResponse.class);
+ String formatISO8601 =
+ Times.formatISO8601(System.currentTimeMillis() + 5 * 1000);
+ when(response.getApplicationTimeouts()).thenReturn(Collections
+ .singletonMap(ApplicationTimeoutType.LIFETIME, formatISO8601));
+
+ when(client
+ .updateApplicationTimeouts(any(UpdateApplicationTimeoutsRequest.class)))
+ .thenReturn(response);
int result = cli.run(new String[] { "application", "-appId",
applicationId.toString(), "-updateLifetime", "10" });
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UpdateApplicationTimeoutsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UpdateApplicationTimeoutsResponsePBImpl.java
index 74f1715..0c94f97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UpdateApplicationTimeoutsResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UpdateApplicationTimeoutsResponsePBImpl.java
@@ -18,10 +18,19 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationUpdateTimeoutMapProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateApplicationTimeoutsResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateApplicationTimeoutsResponseProtoOrBuilder;
import com.google.protobuf.TextFormat;
@@ -33,6 +42,7 @@
UpdateApplicationTimeoutsResponseProto.getDefaultInstance();
UpdateApplicationTimeoutsResponseProto.Builder builder = null;
boolean viaProto = false;
+ private Map applicationTimeouts = null;
public UpdateApplicationTimeoutsResponsePBImpl() {
builder = UpdateApplicationTimeoutsResponseProto.newBuilder();
@@ -45,11 +55,34 @@ public UpdateApplicationTimeoutsResponsePBImpl(
}
public UpdateApplicationTimeoutsResponseProto getProto() {
+ mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = UpdateApplicationTimeoutsResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationTimeouts != null) {
+ addApplicationTimeouts();
+ }
+ }
+
@Override
public int hashCode() {
return getProto().hashCode();
@@ -70,4 +103,79 @@ public boolean equals(Object other) {
public String toString() {
return TextFormat.shortDebugString(getProto());
}
+
+ @Override
+ public Map getApplicationTimeouts() {
+ initApplicationTimeout();
+ return this.applicationTimeouts;
+ }
+
+ private void initApplicationTimeout() {
+ if (this.applicationTimeouts != null) {
+ return;
+ }
+ UpdateApplicationTimeoutsResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List lists =
+ p.getApplicationTimeoutsList();
+ this.applicationTimeouts =
+ new HashMap(lists.size());
+ for (ApplicationUpdateTimeoutMapProto timeoutProto : lists) {
+ this.applicationTimeouts.put(
+ ProtoUtils
+ .convertFromProtoFormat(timeoutProto.getApplicationTimeoutType()),
+ timeoutProto.getExpireTime());
+ }
+ }
+
+ @Override
+ public void setApplicationTimeouts(
+ Map appTimeouts) {
+ if (appTimeouts == null) {
+ return;
+ }
+ initApplicationTimeout();
+ this.applicationTimeouts.clear();
+ this.applicationTimeouts.putAll(appTimeouts);
+ }
+
+ private void addApplicationTimeouts() {
+ maybeInitBuilder();
+ builder.clearApplicationTimeouts();
+ if (applicationTimeouts == null) {
+ return;
+ }
+ Iterable extends ApplicationUpdateTimeoutMapProto> values =
+ new Iterable() {
+
+ @Override
+ public Iterator iterator() {
+ return new Iterator() {
+ private Iterator iterator =
+ applicationTimeouts.keySet().iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public ApplicationUpdateTimeoutMapProto next() {
+ ApplicationTimeoutType key = iterator.next();
+ return ApplicationUpdateTimeoutMapProto.newBuilder()
+ .setExpireTime(applicationTimeouts.get(key))
+ .setApplicationTimeoutType(
+ ProtoUtils.convertToProtoFormat(key))
+ .build();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ this.builder.addAllApplicationTimeouts(values);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
index 785ed04..c28a42b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
@@ -106,6 +106,41 @@
+
+ yarn.scheduler.capacity.root.default.maximum-application-lifetime
+
+ -1
+
+ Maximum lifetime of an application which is submitted to a queue
+ in seconds. Any value less than or equal to zero will be considered as
+ disabled.
+ This will be a hard time limit for all applications in this
+ queue. If positive value is configured then any application submitted
+ to this queue will be killed after exceeds the configured lifetime.
+ User can also specify lifetime per application basis in
+ application submission context. But user lifetime will be
+ overridden if it exceeds queue maximum lifetime. It is point-in-time
+ configuration.
+ Note : Configuring too low value will result in killing application
+ sooner. This feature is applicable only for leaf queue.
+
+
+
+
+ yarn.scheduler.capacity.root.default.default-application-lifetime
+
+ -1
+
+ Default lifetime of an application which is submitted to a queue
+ in seconds. Any value less than or equal to zero will be considered as
+ disabled.
+ If the user has not submitted application with lifetime value then this
+ value will be taken. It is point-in-time configuration.
+ Note : Default lifetime can't exceed maximum lifetime. This feature is
+ applicable only for leaf queue.
+
+
+
yarn.scheduler.capacity.node-locality-delay40
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index e6c25ad..df38893 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1691,6 +1691,7 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts(
RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
AuditConstants.UPDATE_APP_TIMEOUTS, "ClientRMService",
applicationId);
+ response.setApplicationTimeouts(applicationTimeouts);
return response;
}
String msg =
@@ -1702,7 +1703,8 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts(
}
try {
- rmAppManager.updateApplicationTimeout(application, applicationTimeouts);
+ applicationTimeouts = rmAppManager.updateApplicationTimeout(application,
+ applicationTimeouts);
} catch (YarnException ex) {
RMAuditLogger.logFailure(callerUGI.getShortUserName(),
AuditConstants.UPDATE_APP_TIMEOUTS, "UNKNOWN", "ClientRMService",
@@ -1712,6 +1714,7 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts(
RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
AuditConstants.UPDATE_APP_TIMEOUTS, "ClientRMService", applicationId);
+ response.setApplicationTimeouts(applicationTimeouts);
return response;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index bcd1a9c..cb2828f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -66,6 +66,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.SettableFuture;
@@ -571,18 +572,41 @@ public void handle(RMAppManagerEvent event) {
}
// transaction method.
- public void updateApplicationTimeout(RMApp app,
+ public Map updateApplicationTimeout(RMApp app,
Map newTimeoutInISO8601Format)
throws YarnException {
ApplicationId applicationId = app.getApplicationId();
synchronized (applicationId) {
if (app.isAppInCompletedStates()) {
- return;
+ return newTimeoutInISO8601Format;
}
Map newExpireTime = RMServerUtils
.validateISO8601AndConvertToLocalTimeEpoch(newTimeoutInISO8601Format);
+ // validation is only for lifetime
+ Long updatedlifetimeInMillis =
+ newExpireTime.get(ApplicationTimeoutType.LIFETIME);
+ if (updatedlifetimeInMillis != null) {
+ long queueMaxLifetimeInSec =
+ scheduler.getMaximumApplicationLifetime(app.getQueue());
+
+ if (queueMaxLifetimeInSec > 0) {
+ if (updatedlifetimeInMillis > (app.getSubmitTime()
+ + queueMaxLifetimeInSec * 1000)) {
+ updatedlifetimeInMillis =
+ app.getSubmitTime() + queueMaxLifetimeInSec * 1000;
+ // cut off to maximum queue lifetime if update lifetime is exceeding
+ // queue lifetime.
+ newExpireTime.put(ApplicationTimeoutType.LIFETIME,
+ updatedlifetimeInMillis);
+
+ newTimeoutInISO8601Format.put(ApplicationTimeoutType.LIFETIME,
+ Times.formatISO8601(updatedlifetimeInMillis.longValue()));
+ }
+ }
+ }
+
SettableFuture
```
+ * Queue lifetime for applications
+
+ The `CapacityScheduler` supports the following parameters to lifetime of an application:
+
+| Property | Description |
+|:---- |:---- |
+| `yarn.scheduler.capacity..maximum-application-lifetime` | Maximum lifetime of an application which is submitted to a queue in seconds. Any value less than or equal to zero will be considered as disabled. This will be a hard time limit for all applications in this queue. If positive value is configured then any application submitted to this queue will be killed after exceeds the configured lifetime. User can also specify lifetime per application basis in application submission context. But user lifetime will be overridden if it exceeds queue maximum lifetime. It is point-in-time configuration. Note : Configuring too low value will result in killing application sooner. This feature is applicable only for leaf queue. |
+| `yarn.scheduler.capacity.root..default-application-lifetime` | Default lifetime of an application which is submitted to a queue in seconds. Any value less than or equal to zero will be considered as disabled. If the user has not submitted application with lifetime value then this value will be taken. It is point-in-time configuration. Note : Default lifetime can't exceed maximum lifetime. This feature is applicable only for leaf queue.|
+
+
###Setup for application priority.
Application priority works only along with FIFO ordering policy. Default ordering policy is FIFO.