diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index f1ebbfe..1c08ac8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -57,6 +57,7 @@ * maxAppAttempts, the application will be failed. * *
  • Optional, application-specific {@link LogAggregationContext}
  • + *
  • Optional, application-specific {@link UtilityContainerLaunchContext}
  • * *

    * @@ -190,6 +191,23 @@ public static ApplicationSubmissionContext newInstance( context.setLogAggregationContext(logAggregationContext); return context; } + + @Public + @Stable + public static ApplicationSubmissionContext newInstance( + ApplicationId applicationId, String applicationName, String queue, + Priority priority, ContainerLaunchContext amContainer, + boolean isUnmanagedAM, boolean cancelTokensWhenComplete, + int maxAppAttempts, Resource resource, String applicationType, + boolean keepContainers, + UtilityContainerLaunchContext utilityContainerLaunchContext) { + ApplicationSubmissionContext context = + newInstance(applicationId, applicationName, queue, priority, + amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts, + resource, applicationType, keepContainers); + context.setUtilityContainerLaunchContext(utilityContainerLaunchContext); + return context; + } /** * Get the ApplicationId of the submitted application. * @return ApplicationId of the submitted application @@ -532,4 +550,25 @@ public abstract void setLogAggregationContext( @Public @Unstable public abstract void setReservationID(ReservationId reservationID); + + /** + * Get UtilityContainerLaunchContext of the application + * + * @return UtilityContainerLaunchContext of the application + */ + @Public + @Stable + public abstract UtilityContainerLaunchContext + getUtilityContainerLaunchContext(); + + /** + * Set UtilityContainerLaunchContext for the application + * + * @param utilityContainerLaunchContext + * for the application + */ + @Public + @Stable + public abstract void setUtilityContainerLaunchContext( + UtilityContainerLaunchContext utilityContainerLaunchContext); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UtilityContainerLaunchContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UtilityContainerLaunchContext.java new file mode 100644 index 0000000..e80c672 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UtilityContainerLaunchContext.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

    UtilityContainerLaunchContext represents all of the + * information needed by the ResourceManager to launch + * the UtilityContainer for an application.

    + * + *

    It includes details such as: + *

    + *

    + */ +@Public +@Unstable +public abstract class UtilityContainerLaunchContext { + + @Public + @Unstable + public static UtilityContainerLaunchContext newInstance( + boolean isUtilityContainerEnabled, long timeout, int maxRetry, + ContainerLaunchContext utilityContainer) { + UtilityContainerLaunchContext context = + Records.newRecord(UtilityContainerLaunchContext.class); + context.setUtilityContainerEnabled(isUtilityContainerEnabled); + context.setTimeout(timeout); + context.setMaxRetry(maxRetry); + context.setUtilityContainerSpec(utilityContainer); + return context; + } + + /** + * Get the flag which indicates whether to enable the UtilityContainer. + * + * @return the flag which indicates whether to enable the UtilityContainer. + */ + @Public + @Unstable + public abstract boolean isUtilityContainerEnabled(); + + /** + * Set the flag which indicates whether to enable the UtilityContainer. + * + *

    + * If the flag is true, after the application is finished/killed/failed, + * the user specified utilityContainer will be launched. + *

    + * @param isUtilityContainerEnabled + * the flag which indicates whether to enable the UtilityContainer. + */ + @Public + @Unstable + public abstract void setUtilityContainerEnabled( + boolean isUtilityContainerEnabled); + + /** + * get the timeout period on how long the UtilityContainer can run. + * + * @return the timeout period on how long the UtilityContainer can run. + */ + @Public + @Unstable + public abstract long getTimeout(); + + /** + * Set the timeout period on how long the UtilityContainer can run. + * + * @param timeout + */ + @Public + @Unstable + public abstract void setTimeout(long timeout); + + /** + * @return the number of max attempts of the utilityContainer can retry. + */ + @Public + @Unstable + public abstract int getMaxRetry(); + + /** + * Set the number of max attempts of the utilityContainer can retry. + * + * @param maxRetry the number of max attempts of the + * utilityContainer can retry. + */ + @Public + @Unstable + public abstract void setMaxRetry(int maxRetry); + + /** + * Get the ContainerLaunchContext to describe the + * UtilityContainer. + * + * @return ContainerLaunchContext for the + * UtilityContainer. + */ + @Public + @Unstable + public abstract ContainerLaunchContext getUtilityContainerSpec(); + + /** + * Set the ContainerLaunchContext to describe the + * UtilityContainer. + * + * @param utilityContainer ContainerLaunchContext for the + * UtilityContainer + */ + @Public + @Unstable + public abstract void setUtilityContainerSpec( + ContainerLaunchContext utilityContainer); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 4e29d2f..b1e5047 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -308,6 +308,7 @@ message ApplicationSubmissionContextProto { optional ReservationIdProto reservation_id = 15; optional string node_label_expression = 16; optional ResourceRequestProto am_container_resource_request = 17; + optional UtilityContainerLaunchContextProto utility_container_launch_context = 18; } message LogAggregationContextProto { @@ -315,6 +316,13 @@ message LogAggregationContextProto { optional string exclude_pattern = 2 [default = ""]; } +message UtilityContainerLaunchContextProto { + optional bool is_utility_container_enabled = 1 [default = false]; + optional int64 time_out = 2 [default = 600000]; + optional int32 max_retry = 3 [default = 0]; + optional ContainerLaunchContextProto utility_container_spec = 4; +} + enum ApplicationAccessTypeProto { APPACCESS_VIEW_APP = 1; APPACCESS_MODIFY_APP = 2; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index fe89f81..b14fc52 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.UtilityContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto; @@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnProtos.UtilityContainerLaunchContextProto; import com.google.common.base.CharMatcher; import com.google.protobuf.TextFormat; @@ -63,6 +65,7 @@ private ResourceRequest amResourceRequest = null; private LogAggregationContext logAggregationContext = null; private ReservationId reservationId = null; + private UtilityContainerLaunchContext utilityContainerLaunchContext = null; public ApplicationSubmissionContextPBImpl() { builder = ApplicationSubmissionContextProto.newBuilder(); @@ -131,6 +134,10 @@ private void mergeLocalToBuilder() { if (this.reservationId != null) { builder.setReservationId(convertToProtoFormat(this.reservationId)); } + if (this.utilityContainerLaunchContext != null) { + builder.setUtilityContainerLaunchContext( + convertToProtoFormat(this.utilityContainerLaunchContext)); + } } private void mergeLocalToProto() { @@ -548,4 +555,37 @@ private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) { private ReservationIdProto convertToProtoFormat(ReservationId t) { return ((ReservationIdPBImpl) t).getProto(); } + + private UtilityContainerLaunchContextPBImpl convertFromProtoFormat( + UtilityContainerLaunchContextProto p) { + return new UtilityContainerLaunchContextPBImpl(p); + } + + private UtilityContainerLaunchContextProto convertToProtoFormat( + UtilityContainerLaunchContext t) { + return ((UtilityContainerLaunchContextPBImpl) t).getProto(); + } + + @Override + public UtilityContainerLaunchContext getUtilityContainerLaunchContext() { + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + if (this.utilityContainerLaunchContext != null) { + return this.utilityContainerLaunchContext; + } // Else via proto + if (!p.hasUtilityContainerLaunchContext()) { + return null; + } + utilityContainerLaunchContext = + convertFromProtoFormat(p.getUtilityContainerLaunchContext()); + return utilityContainerLaunchContext; + } + + @Override + public void setUtilityContainerLaunchContext( + UtilityContainerLaunchContext utilityContainerLaunchContext) { + maybeInitBuilder(); + if (utilityContainerLaunchContext == null) + builder.clearUtilityContainerLaunchContext(); + this.utilityContainerLaunchContext = utilityContainerLaunchContext; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/UtilityContainerLaunchContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/UtilityContainerLaunchContextPBImpl.java new file mode 100644 index 0000000..f59d466 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/UtilityContainerLaunchContextPBImpl.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.UtilityContainerLaunchContext; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto; +import org.apache.hadoop.yarn.proto.YarnProtos.UtilityContainerLaunchContextProto; +import org.apache.hadoop.yarn.proto.YarnProtos.UtilityContainerLaunchContextProtoOrBuilder; + +import com.google.protobuf.TextFormat; + +@Private +@Unstable +public class UtilityContainerLaunchContextPBImpl extends + UtilityContainerLaunchContext { + UtilityContainerLaunchContextProto proto = + UtilityContainerLaunchContextProto.getDefaultInstance(); + UtilityContainerLaunchContextProto.Builder builder = null; + boolean viaProto = false; + + private ContainerLaunchContext utilityContainer = null; + + public UtilityContainerLaunchContextPBImpl() { + builder = UtilityContainerLaunchContextProto.newBuilder(); + } + + public UtilityContainerLaunchContextPBImpl( + UtilityContainerLaunchContextProto proto) { + this.proto = proto; + viaProto = true; + } + + public UtilityContainerLaunchContextProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.utilityContainer != null) { + builder.setUtilityContainerSpec(convertToProtoFormat(this.utilityContainer)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = UtilityContainerLaunchContextProto.newBuilder(proto); + } + viaProto = false; + } + + private ContainerLaunchContextPBImpl convertFromProtoFormat( + ContainerLaunchContextProto p) { + return new ContainerLaunchContextPBImpl(p); + } + + private ContainerLaunchContextProto convertToProtoFormat( + ContainerLaunchContext t) { + return ((ContainerLaunchContextPBImpl)t).getProto(); + } + + @Override + public boolean isUtilityContainerEnabled() { + UtilityContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder; + return p.getIsUtilityContainerEnabled(); + } + + @Override + public void setUtilityContainerEnabled(boolean isUtilityContainerEnabled) { + maybeInitBuilder(); + builder.setIsUtilityContainerEnabled(isUtilityContainerEnabled); + } + + @Override + public long getTimeout() { + UtilityContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder; + return p.getTimeOut(); + } + + @Override + public void setTimeout(long timeout) { + maybeInitBuilder(); + builder.setTimeOut(timeout); + } + + @Override + public int getMaxRetry() { + UtilityContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder; + return p.getMaxRetry(); + } + + @Override + public void setMaxRetry(int maxRetry) { + maybeInitBuilder(); + builder.setMaxRetry(maxRetry); + } + + @Override + public ContainerLaunchContext getUtilityContainerSpec() { + UtilityContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder; + if (this.utilityContainer != null) { + return utilityContainer; + } // Else via proto + if (!p.hasUtilityContainerSpec()) { + return null; + } + utilityContainer = convertFromProtoFormat(p.getUtilityContainerSpec()); + return utilityContainer; + } + + @Override + public void setUtilityContainerSpec(ContainerLaunchContext utilityContainer) { + maybeInitBuilder(); + if (utilityContainer == null) { + builder.clearUtilityContainerSpec(); + } + this.utilityContainer = utilityContainer; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 8b48798..8d7b302 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -199,6 +199,7 @@ public static void setup() throws Exception { generateByNewInstance(ApplicationReport.class); generateByNewInstance(Container.class); generateByNewInstance(ContainerLaunchContext.class); + generateByNewInstance(UtilityContainerLaunchContext.class); generateByNewInstance(ApplicationSubmissionContext.class); generateByNewInstance(ContainerReport.class); generateByNewInstance(ContainerResourceDecrease.class);