diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index 21cd1bb..7b5207f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -60,6 +60,7 @@ * reaches to maxAppAttempts, the application will be failed. * *
ApplicationId of the submitted application.
* @return ApplicationId of the submitted application
@@ -535,4 +554,25 @@ public abstract void setLogAggregationContext(
@Public
@Unstable
public abstract void setReservationID(ReservationId reservationID);
+
+
+ /**
+ * Get CleanupContainer of the application
+ * no cleanup-container will run if this is not passed or is null
+ *
+ * @return CleanupContainer of the application
+ */
+ @Public
+ @Stable
+ public abstract CleanupContainer getCleanupContainer();
+
+ /**
+ * Set CleanupContainer for the application
+ *
+ * @param CleanupContainer for the application
+ */
+ @Public
+ @Stable
+ public abstract void setCleanupContainer(CleanupContainer cleanupContainer);
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CleanupContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CleanupContainer.java
new file mode 100644
index 0000000..7131441
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CleanupContainer.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * CleanupContainer represents all of the
+ * information needed by the ResourceManager to launch
+ * the cleanupContainer for an application.
It includes details such as: + *
ContainerLaunchContext to describe the
+ * CleanupContainer.
+ *
+ * @return ContainerLaunchContext for the
+ * CleanupContainer.
+ */
+ @Public
+ @Unstable
+ public abstract ContainerLaunchContext getLaunchContext();
+
+ /**
+ * Set the ContainerLaunchContext to describe the
+ * CleanupContainer.
+ *
+ * @param cleanupContainer ContainerLaunchContext for the
+ * CleanupContainer
+ */
+ @Public
+ @Unstable
+ public abstract void setLaunchContext(
+ ContainerLaunchContext launchContext);
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7a05d13..5408f24 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -287,7 +287,22 @@ private static void addDeprecatedKeys() {
public static final String RM_AM_MAX_ATTEMPTS =
RM_PREFIX + "am.max-attempts";
public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2;
-
+
+ /**
+ * The maximum number of attempts for cleanup-container.
+ * It's a global setting for all cleanup Containers.
+ */
+ public static final String RM_CLEAN_UP_CONTAINER_MAX_ATTEMPTS =
+ RM_PREFIX + "cleanup-container.max-attempts";
+ public static final int DEFAULT_RM_CLEAN_UP_CONTAINER_MAX_ATTEMPTS = 2;
+
+ /**
+ * The duration that a cleanup container can run.
+ */
+ public static final String RM_CLEAN_UP_CONTAINER_TIMEOUT_MS =
+ RM_PREFIX + "cleanup-container.timeout-ms";
+ public static final long DEFAULT_RM_CLEAN_UP_CONTAINER_TIMEOUT_MS = 600000;
+
/** The keytab for the resource manager.*/
public static final String RM_KEYTAB =
RM_PREFIX + "keytab";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 4095676..07d9dee 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -332,6 +332,7 @@ message ApplicationSubmissionContextProto {
optional ReservationIdProto reservation_id = 15;
optional string node_label_expression = 16;
optional ResourceRequestProto am_container_resource_request = 17;
+ optional CleanupContainerProto cleanup_container = 18;
}
message LogAggregationContextProto {
@@ -341,6 +342,11 @@ message LogAggregationContextProto {
optional string rolled_logs_exclude_pattern = 4 [default = ".*"];
}
+message CleanupContainerProto {
+ optional int32 max_attempts = 1 [default = 0];
+ optional ContainerLaunchContextProto launch_context = 2;
+}
+
enum ApplicationAccessTypeProto {
APPACCESS_VIEW_APP = 1;
APPACCESS_MODIFY_APP = 2;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index 67e3a84..59d46f0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.CleanupContainer;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -36,6 +37,7 @@
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.CleanupContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
@@ -63,6 +65,7 @@
private ResourceRequest amResourceRequest = null;
private LogAggregationContext logAggregationContext = null;
private ReservationId reservationId = null;
+ private CleanupContainer cleanupContainer = null;
public ApplicationSubmissionContextPBImpl() {
builder = ApplicationSubmissionContextProto.newBuilder();
@@ -131,6 +134,9 @@ private void mergeLocalToBuilder() {
if (this.reservationId != null) {
builder.setReservationId(convertToProtoFormat(this.reservationId));
}
+ if (this.cleanupContainer != null) {
+ builder.setCleanupContainer(convertToProtoFormat(this.cleanupContainer));
+ }
}
private void mergeLocalToProto() {
@@ -548,4 +554,38 @@ private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
private ReservationIdProto convertToProtoFormat(ReservationId t) {
return ((ReservationIdPBImpl) t).getProto();
}
+
+ @Override
+ public CleanupContainer getCleanupContainer() {
+ ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.cleanupContainer != null) {
+ return this.cleanupContainer;
+ } // Else via proto
+ if (!p.hasCleanupContainer()) {
+ return null;
+ }
+ cleanupContainer =
+ convertFromProtoFormat(p.getCleanupContainer());
+ return cleanupContainer;
+ }
+
+ @Override
+ public void setCleanupContainer(CleanupContainer cleanupContainer) {
+ maybeInitBuilder();
+ if (cleanupContainer == null)
+ builder.clearCleanupContainer();
+ this.cleanupContainer = cleanupContainer;
+
+ }
+
+ private CleanupContainerPBImpl convertFromProtoFormat(
+ CleanupContainerProto p) {
+ return new CleanupContainerPBImpl(p);
+ }
+
+ private CleanupContainerProto convertToProtoFormat(
+ CleanupContainer t) {
+ return ((CleanupContainerPBImpl) t).getProto();
+ }
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CleanupContainerPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CleanupContainerPBImpl.java
new file mode 100644
index 0000000..0db16db
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CleanupContainerPBImpl.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.CleanupContainer;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.proto.YarnProtos.CleanupContainerProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.CleanupContainerProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto;
+
+import com.google.protobuf.TextFormat;
+
+@Private
+@Unstable
+public class CleanupContainerPBImpl extends CleanupContainer {
+
+ CleanupContainerProto proto =
+ CleanupContainerProto.getDefaultInstance();
+ CleanupContainerProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ContainerLaunchContext launchContext = null;
+
+ public CleanupContainerPBImpl() {
+ builder = CleanupContainerProto.newBuilder();
+ }
+
+ public CleanupContainerPBImpl(
+ CleanupContainerProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public CleanupContainerProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.launchContext != null) {
+ builder.setLaunchContext(convertToProtoFormat(this.launchContext));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = CleanupContainerProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private ContainerLaunchContextPBImpl convertFromProtoFormat(
+ ContainerLaunchContextProto p) {
+ return new ContainerLaunchContextPBImpl(p);
+ }
+
+ private ContainerLaunchContextProto convertToProtoFormat(
+ ContainerLaunchContext t) {
+ return ((ContainerLaunchContextPBImpl)t).getProto();
+ }
+
+ @Override
+ public int getMaxAttempts() {
+ CleanupContainerProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMaxAttempts();
+ }
+
+ @Override
+ public void setMaxAttempts(int maxAttempts) {
+ maybeInitBuilder();
+ builder.setMaxAttempts(maxAttempts);
+ }
+
+ @Override
+ public ContainerLaunchContext getLaunchContext() {
+ CleanupContainerProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.launchContext != null) {
+ return launchContext;
+ } // Else via proto
+ if (!p.hasLaunchContext()) {
+ return null;
+ }
+ launchContext = convertFromProtoFormat(p.getLaunchContext());
+ return launchContext;
+ }
+
+ @Override
+ public void setLaunchContext(ContainerLaunchContext launchContext) {
+ maybeInitBuilder();
+ if (launchContext == null) {
+ builder.clearLaunchContext();
+ }
+ this.launchContext = launchContext;
+ }
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 8dacd3b..ae2b9f1 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -109,6 +109,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.CleanupContainer;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -465,6 +466,7 @@ public static void setup() throws Exception {
generateByNewInstance(ApplicationReport.class);
generateByNewInstance(Container.class);
generateByNewInstance(ContainerLaunchContext.class);
+ generateByNewInstance(CleanupContainer.class);
generateByNewInstance(ApplicationSubmissionContext.class);
generateByNewInstance(ContainerReport.class);
generateByNewInstance(ContainerResourceDecrease.class);