From 391a8b19fcbd5f64ebdd68ef7cbd210ff05712c8 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Thu, 17 Nov 2016 22:46:56 +0530 Subject: [PATCH] YARN-4506 --- .../hadoop/yarn/api/records/ApplicationReport.java | 10 ++ .../yarn/api/records/ApplicationTimeout.java | 51 ++++++++++ .../src/main/proto/yarn_protos.proto | 7 ++ .../apache/hadoop/yarn/client/api/YarnClient.java | 11 +++ .../yarn/client/api/impl/YarnClientImpl.java | 9 ++ .../hadoop/yarn/client/cli/ApplicationCLI.java | 40 +++++++- .../apache/hadoop/yarn/client/cli/TestYarnCLI.java | 34 +++++++ .../records/impl/pb/ApplicationReportPBImpl.java | 81 ++++++++++++++++ .../records/impl/pb/ApplicationTimeoutPBImpl.java | 103 +++++++++++++++++++++ .../server/resourcemanager/rmapp/RMAppImpl.java | 19 ++++ .../rmapp/TestApplicationLifetimeMonitor.java | 31 ++++++- 11 files changed, 393 insertions(+), 3 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationTimeoutPBImpl.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 33116a4..9db83c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; +import java.util.List; import java.util.Set; /** @@ -447,4 +448,13 @@ public abstract void setLogAggregationStatus( @Unstable public abstract void setAmNodeLabelExpression(String amNodeLabelExpression); + + @Public + @Unstable + public abstract List getApplicationTimeoutList(); + + @Private + @Unstable + public abstract void setApplicationTimeoutList( + List timeouts); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java new file mode 100644 index 0000000..828ebe4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationTimeout.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +@Public +@Unstable +public abstract class ApplicationTimeout { + + @Public + @Unstable + public static ApplicationTimeout newInstance(ApplicationTimeoutType type, + String expire_time, long remainingTime) { + ApplicationTimeout timeouts = Records.newRecord(ApplicationTimeout.class); + timeouts.setAppTimeoutType(type); + timeouts.setExpireTime(expire_time); + timeouts.setRemainingTime(remainingTime); + return timeouts; + } + + public abstract ApplicationTimeoutType getAppTimeoutType(); + + public abstract void setAppTimeoutType(ApplicationTimeoutType type); + + public abstract String getExpireTime(); + + public abstract void setExpireTime(String expire_time); + + public abstract long getRemainingTime(); + + public abstract void setRemainingTime(long remainingTime); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index cb37126..43a661f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -214,6 +214,13 @@ message ApplicationReportProto { optional PriorityProto priority = 23; optional string appNodeLabelExpression = 24; optional string amNodeLabelExpression = 25; + repeated ApplicationTimeoutProto application_timeouts = 26; +} + +message ApplicationTimeoutProto { + required ApplicationTimeoutTypeProto application_timeout_type = 1; + optional string expire_time = 2; + optional int64 remaining_time = 3; } enum LogAggregationStatusProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 4e0ba2c..0bf1643 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -844,4 +846,13 @@ public abstract Priority updateApplicationPriority( */ public abstract void signalToContainer(ContainerId containerId, SignalContainerCommand command) throws YarnException, IOException; + + @Public + @Unstable + public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( + UpdateApplicationTimeoutsRequest request) + throws YarnException, IOException { + throw new UnsupportedOperationException("The sub-class extending " + + YarnClient.class.getName() + " is expected to implement this !"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 57f50c4..b4c7aa7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -84,6 +84,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -904,4 +906,11 @@ public void signalToContainer(ContainerId containerId, SignalContainerRequest.newInstance(containerId, command); rmClient.signalToContainer(request); } + + @Override + public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( + UpdateApplicationTimeoutsRequest request) + throws YarnException, IOException { + return rmClient.updateApplicationTimeouts(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index b841601..cfc7fa1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -24,6 +24,7 @@ import java.nio.charset.Charset; import java.text.DecimalFormat; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; import java.util.List; @@ -39,11 +40,14 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.Priority; @@ -84,6 +88,7 @@ public static final String CONTAINER = "container"; public static final String APP_ID = "appId"; public static final String UPDATE_PRIORITY = "updatePriority"; + public static final String UPDATE_TIMEOUT = "updateTimeout"; private boolean allAppStates; @@ -137,6 +142,9 @@ public int run(String[] args) throws Exception { opts.addOption(appTagOpt); opts.addOption(APP_ID, true, "Specify Application Id to be operated"); opts.addOption(UPDATE_PRIORITY, true, + "update timeout of an application from NOW. ApplicationId can be" + + " passed using 'appId' option."); + opts.addOption(UPDATE_TIMEOUT, true, "update priority of an application. ApplicationId can be" + " passed using 'appId' option."); Option killOpt = new Option(KILL_CMD, true, "Kills the application. " @@ -150,6 +158,7 @@ public int run(String[] args) throws Exception { opts.getOption(STATUS_CMD).setArgName("Application ID"); opts.getOption(APP_ID).setArgName("Application ID"); opts.getOption(UPDATE_PRIORITY).setArgName("Priority"); + opts.getOption(UPDATE_TIMEOUT).setArgName("Timeout"); } else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { title = APPLICATION_ATTEMPT; opts.addOption(STATUS_CMD, true, @@ -296,6 +305,13 @@ public int run(String[] args) throws Exception { } updateApplicationPriority(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(UPDATE_PRIORITY)); + } else if (cliParser.hasOption(UPDATE_TIMEOUT)) { + if (!cliParser.hasOption(APP_ID)) { + printUsage(title, opts); + return exitCode; + } + updateApplicationTimeout(cliParser.getOptionValue(APP_ID), + cliParser.getOptionValue(UPDATE_TIMEOUT)); } else if (cliParser.hasOption(SIGNAL_CMD)) { if (args.length < 3 || args.length > 4) { printUsage(title, opts); @@ -316,6 +332,20 @@ public int run(String[] args) throws Exception { return 0; } + private void updateApplicationTimeout(String applicationId, String timeout) + throws YarnException, IOException { + ApplicationId appId = ApplicationId.fromString(applicationId); + String newTimeout = Times + .format(System.currentTimeMillis() + (Long.parseLong(timeout)) * 1000); + sysout.println("Updating timeout of an application " + applicationId); + UpdateApplicationTimeoutsRequest request = + UpdateApplicationTimeoutsRequest.newInstance(appId, Collections + .singletonMap(ApplicationTimeoutType.LIFETIME, newTimeout)); + client.updateApplicationTimeouts(request); + sysout.println("Successfully updated timeout of an application " + + applicationId + ". New expire time will be " + newTimeout); + } + /** * Signals the containerId * @@ -678,7 +708,15 @@ private int printApplicationReport(String applicationId) appReportStr.print("\tApplication Node Label Expression : "); appReportStr.println(appReport.getAppNodeLabelExpression()); appReportStr.print("\tAM container Node Label Expression : "); - appReportStr.print(appReport.getAmNodeLabelExpression()); + appReportStr.println(appReport.getAmNodeLabelExpression()); + if (!appReport.getApplicationTimeoutList().isEmpty()) { + ApplicationTimeout timeout = + appReport.getApplicationTimeoutList().iterator().next(); + appReportStr.print("\tApplication Expire Time : "); + appReportStr.println(timeout.getExpireTime()); + appReportStr.print("\tApplication Remaining Time : "); + appReportStr.print(timeout.getRemainingTime()); + } } else { appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 09dec89..87363a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -36,6 +36,7 @@ import java.io.PrintWriter; import java.io.UnsupportedEncodingException; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.EnumSet; import java.util.HashSet; @@ -46,11 +47,14 @@ import org.apache.commons.cli.Options; import org.apache.commons.lang.time.DateFormatUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -124,6 +128,11 @@ public void testGetApplicationReport() throws Exception { null, null, false, Priority.newInstance(0), "high-mem", "high-mem"); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); newApplicationReport.setPriority(Priority.newInstance(0)); + ApplicationTimeout timeout = ApplicationTimeout + .newInstance(ApplicationTimeoutType.LIFETIME, "N/A", -1); + newApplicationReport + .setApplicationTimeoutList(Collections.singletonList(timeout)); + when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); int result = cli.run(new String[] { "application", "-status", applicationId.toString() }); @@ -155,6 +164,8 @@ public void testGetApplicationReport() throws Exception { pw.println("\tUnmanaged Application : false"); pw.println("\tApplication Node Label Expression : high-mem"); pw.println("\tAM container Node Label Expression : high-mem"); + pw.println("\tApplication Expire Time : N/A"); + pw.println("\tApplication Remaining Time : -1"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); @@ -2074,4 +2085,27 @@ public void testAppAttemptReportWhileContainerIsNotAssigned() applicationId.toString() }); assertEquals(0, result); } + + @Test(timeout = 60000) + public void testUpdateApplicationTimeout() throws Exception { + ApplicationCLI cli = createAndGetAppCLI(); + ApplicationId applicationId = ApplicationId.newInstance(1234, 6); + + ApplicationReport appReport = ApplicationReport.newInstance(applicationId, + ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", + "appname", "host", 124, null, YarnApplicationState.RUNNING, + "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, + "N/A", 0.53789f, "YARN", null); + ApplicationTimeout timeout = ApplicationTimeout + .newInstance(ApplicationTimeoutType.LIFETIME, "N/A", -1); + appReport.setApplicationTimeoutList(Collections.singletonList(timeout)); + when(client.getApplicationReport(any(ApplicationId.class))) + .thenReturn(appReport); + + int result = cli.run(new String[] { "application", "-appId", + applicationId.toString(), "-updateTimeout", "10" }); + Assert.assertEquals(result, 0); + verify(client) + .updateApplicationTimeouts(any(UpdateApplicationTimeoutsRequest.class)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 1072815..8bb9bc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.Priority; @@ -35,6 +36,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutProto; import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto; @@ -42,7 +44,10 @@ import com.google.protobuf.TextFormat; +import java.util.ArrayList; import java.util.HashSet; +import java.util.Iterator; +import java.util.List; import java.util.Set; @Private @@ -58,6 +63,7 @@ private Token amRmToken = null; private Set applicationTags = null; private Priority priority = null; + private List applicationTimeoutList = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -492,6 +498,9 @@ private void mergeLocalToBuilder() { builder.getPriority())) { builder.setPriority(convertToProtoFormat(this.priority)); } + if (this.applicationTimeoutList != null) { + addLocalApplicationTimeoutToProto(); + } } private void mergeLocalToProto() { @@ -668,4 +677,76 @@ public void setAmNodeLabelExpression(String amNodeLabelExpression) { } builder.setAmNodeLabelExpression((amNodeLabelExpression)); } + + @Override + public List getApplicationTimeoutList() { + initLocalApplicationsList(); + return this.applicationTimeoutList; + } + + private void initLocalApplicationsList() { + if (this.applicationTimeoutList != null) { + return; + } + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getApplicationTimeoutsList(); + this.applicationTimeoutList = new ArrayList(); + + for (ApplicationTimeoutProto a : list) { + this.applicationTimeoutList.add(convertFromProtoFormat(a)); + } + } + + private void addLocalApplicationTimeoutToProto() { + maybeInitBuilder(); + builder.clearApplicationTimeouts(); + if (applicationTimeoutList == null) + return; + Iterable iterable = + new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = + applicationTimeoutList.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ApplicationTimeoutProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllApplicationTimeouts(iterable); + } + + private ApplicationTimeoutPBImpl convertFromProtoFormat( + ApplicationTimeoutProto p) { + return new ApplicationTimeoutPBImpl(p); + } + + private ApplicationTimeoutProto convertToProtoFormat(ApplicationTimeout t) { + return ((ApplicationTimeoutPBImpl) t).getProto(); + } + + @Override + public void setApplicationTimeoutList(List timeouts) { + maybeInitBuilder(); + if (timeouts == null) + builder.clearApplicationTimeouts(); + this.applicationTimeoutList = timeouts; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationTimeoutPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationTimeoutPBImpl.java new file mode 100644 index 0000000..c2a8e65 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationTimeoutPBImpl.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutProtoOrBuilder; + +public class ApplicationTimeoutPBImpl extends ApplicationTimeout { + + ApplicationTimeoutProto proto = ApplicationTimeoutProto.getDefaultInstance(); + ApplicationTimeoutProto.Builder builder = null; + boolean viaProto = false; + + public ApplicationTimeoutPBImpl() { + builder = ApplicationTimeoutProto.newBuilder(); + } + + public ApplicationTimeoutPBImpl(ApplicationTimeoutProto proto) { + this.proto = proto; + viaProto = true; + } + + public ApplicationTimeoutProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationTimeoutProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public ApplicationTimeoutType getAppTimeoutType() { + ApplicationTimeoutProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationTimeoutType()) { + return null; + } + return ProtoUtils.convertFromProtoFormat(p.getApplicationTimeoutType()); + } + + @Override + public void setAppTimeoutType(ApplicationTimeoutType type) { + maybeInitBuilder(); + if (type == null) { + builder.clearApplicationTimeoutType(); + return; + } + builder.setApplicationTimeoutType(ProtoUtils.convertToProtoFormat(type)); + } + + @Override + public String getExpireTime() { + ApplicationTimeoutProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasExpireTime()) { + return null; + } + return p.getExpireTime(); + } + + @Override + public void setExpireTime(String expire_time) { + maybeInitBuilder(); + if (expire_time == null) { + builder.clearExpireTime(); + return; + } + builder.setExpireTime(expire_time); + } + + @Override + public long getRemainingTime() { + ApplicationTimeoutProtoOrBuilder p = viaProto ? proto : builder; + return p.getRemainingTime(); + } + + @Override + public void setRemainingTime(long remainingTime) { + maybeInitBuilder(); + builder.setRemainingTime(remainingTime); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index a376311..9759885 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; @@ -111,6 +112,7 @@ import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -771,6 +773,12 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, DUMMY_APPLICATION_ATTEMPT_NUMBER); } + if (this.applicationTimeouts.isEmpty()) { + + } else { + + } + ApplicationReport report = BuilderUtils.newApplicationReport( this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, @@ -782,6 +790,17 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, report.setUnmanagedApp(submissionContext.getUnmanagedAM()); report.setAppNodeLabelExpression(getAppNodeLabelExpression()); report.setAmNodeLabelExpression(getAmNodeLabelExpression()); + + if (!this.applicationTimeouts.isEmpty()) { + ApplicationTimeout timeout = ApplicationTimeout + .newInstance(ApplicationTimeoutType.LIFETIME, UNAVAILABLE, -1); + long timeoutInMillis = applicationTimeouts + .get(ApplicationTimeoutType.LIFETIME).longValue(); + timeout.setExpireTime(Times.formatISO8601(timeoutInMillis)); + timeout.setRemainingTime( + Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0)); + report.setApplicationTimeoutList(Collections.singletonList(timeout)); + } return report; } finally { this.readLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java index e803a88..dfb9b4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java @@ -24,18 +24,24 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -101,8 +107,9 @@ public void testApplicationLifetimeMonitor() throws Exception { new HashMap(); long newLifetime = 10L; // update 10L seconds more to timeout - updateTimeout.put(ApplicationTimeoutType.LIFETIME, - Times.formatISO8601(System.currentTimeMillis() + newLifetime * 1000)); + String formatISO8601 = + Times.formatISO8601(System.currentTimeMillis() + newLifetime * 1000); + updateTimeout.put(ApplicationTimeoutType.LIFETIME, formatISO8601); UpdateApplicationTimeoutsRequest request = UpdateApplicationTimeoutsRequest.newInstance(app2.getApplicationId(), updateTimeout); @@ -124,6 +131,26 @@ public void testApplicationLifetimeMonitor() throws Exception { Assert.assertTrue("Application lifetime value not updated", afterUpdate > beforeUpdate); + // verify for application report. + RecordFactory recordFactory = + RecordFactoryProvider.getRecordFactory(null); + GetApplicationReportRequest appRequest = + recordFactory.newRecordInstance(GetApplicationReportRequest.class); + request.setApplicationId(app2.getApplicationId()); + List applicationTimeoutList = rm.getRMContext() + .getClientRMService().getApplicationReport(appRequest) + .getApplicationReport().getApplicationTimeoutList(); + Assert.assertTrue("Application Timeout list are empty.", + !applicationTimeoutList.isEmpty()); + ApplicationTimeout timeout = applicationTimeoutList.iterator().next(); + Assert.assertEquals("Application timeout Type is incorrect.", + ApplicationTimeoutType.LIFETIME.toString(), + timeout.getAppTimeoutType().toString()); + Assert.assertEquals("Application timeout string is incorrect.", + formatISO8601, timeout.getExpireTime()); + Assert.assertTrue("Application remaining time is incorrect", + timeout.getRemainingTime() > 0); + rm.waitForState(app2.getApplicationId(), RMAppState.KILLED); // verify for app killed with updated lifetime Assert.assertTrue("Application killed before lifetime value", -- 2.7.4 (Apple Git-66)