diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index 21cd1bb..457c940 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -60,6 +60,7 @@ * reaches to maxAppAttempts, the application will be failed. * *
  • Optional, application-specific {@link LogAggregationContext}
  • + *
  • Optional, application-specific {@link CleanupContainer}
  • * * * @see ContainerLaunchContext @@ -193,6 +194,24 @@ public static ApplicationSubmissionContext newInstance( context.setLogAggregationContext(logAggregationContext); return context; } + + @Public + @Stable + public static ApplicationSubmissionContext newInstance( + ApplicationId applicationId, String applicationName, String queue, + Priority priority, ContainerLaunchContext amContainer, + boolean isUnmanagedAM, boolean cancelTokensWhenComplete, + int maxAppAttempts, Resource resource, String applicationType, + boolean keepContainers, + CleanupContainer cleanupContainer) { + ApplicationSubmissionContext context = + newInstance(applicationId, applicationName, queue, priority, + amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts, + resource, applicationType, keepContainers); + context.setCleanupContainer(cleanupContainer); + return context; + } + /** * Get the ApplicationId of the submitted application. * @return ApplicationId of the submitted application @@ -535,4 +554,24 @@ public abstract void setLogAggregationContext( @Public @Unstable public abstract void setReservationID(ReservationId reservationID); + + /** + * Get CleanupContainer of the application + * no cleanup-container will run if this is not passed or is null + * + * @return CleanupContainer of the application + */ + @Public + @Stable + public abstract CleanupContainer getCleanupContainer(); + + /** + * Set CleanupContainer for the application + * + * @param cleanupContainer for the application + */ + @Public + @Stable + public abstract void setCleanupContainer(CleanupContainer cleanupContainer); + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CleanupContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CleanupContainer.java new file mode 100644 index 0000000..616197f --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CleanupContainer.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.Records; + +/** + *

    CleanupContainer represents all of the + * information needed by the ResourceManager to launch + * the cleanupContainer for an application.

    + * + *

    It includes details such as: + *

    + *

    + */ + +@Public +@Unstable +public abstract class CleanupContainer { + + @Public + @Unstable + public static CleanupContainer newInstance(int maxAttempts, + ContainerLaunchContext launchContext) { + CleanupContainer container = Records.newRecord(CleanupContainer.class); + container.setMaxAttempts(maxAttempts); + container.setLaunchContext(launchContext); + return container; + } + + /** + * @return the number of max attempts of the CleanupContainer can be submitted. + */ + @Public + @Unstable + public abstract int getMaxAttempts(); + + /** + * Set the number of max attempts of the CleanupContainer to be submitted. + * WARNING: it should be no larger than the global number of max attempts + * in the Yarn configuration. + * @param maxAttempts the number of max attempts of the + * CleanupContainer to be submitted. + */ + @Public + @Unstable + public abstract void setMaxAttempts(int maxAttempts); + + /** + * Get the ContainerLaunchContext to describe the + * CleanupContainer. + * + * @return ContainerLaunchContext for the + * CleanupContainer. + */ + @Public + @Unstable + public abstract ContainerLaunchContext getLaunchContext(); + + /** + * Set the ContainerLaunchContext to describe the + * CleanupContainer. + * + * @param launchContext ContainerLaunchContext for the + * CleanupContainer + */ + @Public + @Unstable + public abstract void setLaunchContext( + ContainerLaunchContext launchContext); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e06ec1c..afb3d7a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -299,7 +299,22 @@ private static void addDeprecatedKeys() { public static final String RM_AM_MAX_ATTEMPTS = RM_PREFIX + "am.max-attempts"; public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2; - + + /** + * The maximum number of attempts for cleanup-container. + * It's a global setting for all cleanup Containers. + */ + public static final String RM_CLEAN_UP_CONTAINER_MAX_ATTEMPTS = + RM_PREFIX + "cleanup-container.max-attempts"; + public static final int DEFAULT_RM_CLEAN_UP_CONTAINER_MAX_ATTEMPTS = 2; + + /** + * The duration that a cleanup container can run. + */ + public static final String RM_CLEAN_UP_CONTAINER_TIMEOUT_MS = + RM_PREFIX + "cleanup-container.timeout-ms"; + public static final long DEFAULT_RM_CLEAN_UP_CONTAINER_TIMEOUT_MS = 600000; + /** The keytab for the resource manager.*/ public static final String RM_KEYTAB = RM_PREFIX + "keytab"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index f801409..32095f7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -329,6 +329,7 @@ message ApplicationSubmissionContextProto { optional ReservationIdProto reservation_id = 15; optional string node_label_expression = 16; optional ResourceRequestProto am_container_resource_request = 17; + optional CleanupContainerProto cleanup_container = 18; } message LogAggregationContextProto { @@ -338,6 +339,11 @@ message LogAggregationContextProto { optional string rolled_logs_exclude_pattern = 4 [default = ".*"]; } +message CleanupContainerProto { + optional int32 max_attempts = 1 [default = 0]; + optional ContainerLaunchContextProto launch_context = 2; +} + enum ApplicationAccessTypeProto { APPACCESS_VIEW_APP = 1; APPACCESS_MODIFY_APP = 2; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index 67e3a84..510d390 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -26,6 +26,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.CleanupContainer; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.Priority; @@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.CleanupContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto; @@ -63,6 +65,7 @@ private ResourceRequest amResourceRequest = null; private LogAggregationContext logAggregationContext = null; private ReservationId reservationId = null; + private CleanupContainer cleanupContainer = null; public ApplicationSubmissionContextPBImpl() { builder = ApplicationSubmissionContextProto.newBuilder(); @@ -131,6 +134,9 @@ private void mergeLocalToBuilder() { if (this.reservationId != null) { builder.setReservationId(convertToProtoFormat(this.reservationId)); } + if (this.cleanupContainer != null) { + builder.setCleanupContainer(convertToProtoFormat(this.cleanupContainer)); + } } private void mergeLocalToProto() { @@ -548,4 +554,38 @@ private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) { private ReservationIdProto convertToProtoFormat(ReservationId t) { return ((ReservationIdPBImpl) t).getProto(); } -} + + @Override + public CleanupContainer getCleanupContainer() { + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + if (this.cleanupContainer != null) { + return this.cleanupContainer; + } // Else via proto + if (!p.hasCleanupContainer()) { + return null; + } + cleanupContainer = + convertFromProtoFormat(p.getCleanupContainer()); + return cleanupContainer; + } + + @Override + public void setCleanupContainer(CleanupContainer cleanupContainer) { + maybeInitBuilder(); + if (cleanupContainer == null) + builder.clearCleanupContainer(); + this.cleanupContainer = cleanupContainer; + + } + + private CleanupContainerPBImpl convertFromProtoFormat( + CleanupContainerProto p) { + return new CleanupContainerPBImpl(p); + } + + private CleanupContainerProto convertToProtoFormat( + CleanupContainer t) { + return ((CleanupContainerPBImpl) t).getProto(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CleanupContainerPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CleanupContainerPBImpl.java new file mode 100644 index 0000000..0db16db --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CleanupContainerPBImpl.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.CleanupContainer; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.proto.YarnProtos.CleanupContainerProto; +import org.apache.hadoop.yarn.proto.YarnProtos.CleanupContainerProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto; + +import com.google.protobuf.TextFormat; + +@Private +@Unstable +public class CleanupContainerPBImpl extends CleanupContainer { + + CleanupContainerProto proto = + CleanupContainerProto.getDefaultInstance(); + CleanupContainerProto.Builder builder = null; + boolean viaProto = false; + + private ContainerLaunchContext launchContext = null; + + public CleanupContainerPBImpl() { + builder = CleanupContainerProto.newBuilder(); + } + + public CleanupContainerPBImpl( + CleanupContainerProto proto) { + this.proto = proto; + viaProto = true; + } + + public CleanupContainerProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.launchContext != null) { + builder.setLaunchContext(convertToProtoFormat(this.launchContext)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = CleanupContainerProto.newBuilder(proto); + } + viaProto = false; + } + + private ContainerLaunchContextPBImpl convertFromProtoFormat( + ContainerLaunchContextProto p) { + return new ContainerLaunchContextPBImpl(p); + } + + private ContainerLaunchContextProto convertToProtoFormat( + ContainerLaunchContext t) { + return ((ContainerLaunchContextPBImpl)t).getProto(); + } + + @Override + public int getMaxAttempts() { + CleanupContainerProtoOrBuilder p = viaProto ? proto : builder; + return p.getMaxAttempts(); + } + + @Override + public void setMaxAttempts(int maxAttempts) { + maybeInitBuilder(); + builder.setMaxAttempts(maxAttempts); + } + + @Override + public ContainerLaunchContext getLaunchContext() { + CleanupContainerProtoOrBuilder p = viaProto ? proto : builder; + if (this.launchContext != null) { + return launchContext; + } // Else via proto + if (!p.hasLaunchContext()) { + return null; + } + launchContext = convertFromProtoFormat(p.getLaunchContext()); + return launchContext; + } + + @Override + public void setLaunchContext(ContainerLaunchContext launchContext) { + maybeInitBuilder(); + if (launchContext == null) { + builder.clearLaunchContext(); + } + this.launchContext = launchContext; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 8dacd3b..ae2b9f1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -109,6 +109,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.CleanupContainer; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -465,6 +466,7 @@ public static void setup() throws Exception { generateByNewInstance(ApplicationReport.class); generateByNewInstance(Container.class); generateByNewInstance(ContainerLaunchContext.class); + generateByNewInstance(CleanupContainer.class); generateByNewInstance(ApplicationSubmissionContext.class); generateByNewInstance(ContainerReport.class); generateByNewInstance(ContainerResourceDecrease.class);