From 9c75484da1d44764feb499c34f743893b4d873a6 Mon Sep 17 00:00:00 2001 From: Garry Weng Date: Fri, 11 Sep 2020 02:37:23 +0000 Subject: [PATCH] Implement application group feature --- .../org/apache/hadoop/http/JettyUtils.java | 35 ++ .../hadoop/mapred/ResourceMgrDelegate.java | 19 + .../hadoop/mapred/TestClientRedirect.java | 27 ++ hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 8 +- .../yarn/api/ApplicationClientProtocol.java | 56 +++ .../CreateApplicationGroupRequest.java | 83 +++++ .../CreateApplicationGroupResponse.java | 45 +++ .../ListApplicationGroupsRequest.java | 44 +++ .../ListApplicationGroupsResponse.java | 63 ++++ .../TerminateApplicationGroupRequest.java | 57 +++ .../TerminateApplicationGroupResponse.java | 48 +++ .../hadoop/yarn/api/records/AppGroupACL.java | 48 +++ .../yarn/api/records/ApplicationGroup.java | 120 +++++++ .../records/YarnApplicationGroupState.java | 33 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 13 + .../ApplicationGroupNotFoundException.java | 42 +++ .../proto/applicationclient_protocol.proto | 3 + .../src/main/proto/yarn_protos.proto | 13 + .../src/main/proto/yarn_service_protos.proto | 23 ++ .../hadoop/yarn/client/api/YarnClient.java | 38 ++ .../yarn/client/api/impl/YarnClientImpl.java | 28 ++ .../yarn/client/cli/ApplicationGroupCLI.java | 172 +++++++++ ...ApplicationClientProtocolPBClientImpl.java | 60 ++++ ...pplicationClientProtocolPBServiceImpl.java | 67 ++++ .../CreateApplicationGroupRequestPBImpl.java | 162 +++++++++ .../CreateApplicationGroupResponsePBImpl.java | 74 ++++ .../ListApplicationGroupsRequestPBImpl.java | 75 ++++ .../ListApplicationGroupsResponsePBImpl.java | 180 ++++++++++ ...erminateApplicationGroupRequestPBImpl.java | 124 +++++++ ...rminateApplicationGroupResponsePBImpl.java | 74 ++++ .../impl/pb/ApplicationGroupPBImpl.java | 116 ++++++ .../server/MockResourceManagerFacade.java | 34 ++ .../resourcemanager/AppGroupMonitor.java | 188 ++++++++++ .../resourcemanager/ClientRMService.java | 138 +++++++ .../RMActiveServiceContext.java | 10 + .../resourcemanager/RMAppGroupManager.java | 338 ++++++++++++++++++ .../server/resourcemanager/RMAuditLogger.java | 4 + .../server/resourcemanager/RMContext.java | 3 + .../server/resourcemanager/RMContextImpl.java | 6 + .../resourcemanager/ResourceManager.java | 27 +- .../recovery/FileSystemRMStateStore.java | 20 ++ .../recovery/LeveldbRMStateStore.java | 20 ++ .../recovery/MemoryRMStateStore.java | 30 ++ .../recovery/NullRMStateStore.java | 19 + .../recovery/RMStateRemoveAppGroupEvent.java | 48 +++ .../recovery/RMStateStore.java | 159 ++++++++ .../recovery/RMStateStoreAppGroupEvent.java | 48 +++ .../recovery/RMStateStoreEventType.java | 3 + .../recovery/RMStateUpdateAppGroupEvent.java | 50 +++ .../recovery/ZKRMStateStore.java | 74 ++++ .../records/ApplicationGroupStateData.java | 90 +++++ .../pb/ApplicationGroupStateDataPBImpl.java | 204 +++++++++++ .../resourcemanager/rmapp/RMAppState.java | 8 +- .../rmapp/group/RMAppGroup.java | 93 +++++ .../rmapp/group/RMAppGroupImpl.java | 160 +++++++++ .../resourcemanager/webapp/RMWSConsts.java | 6 + .../resourcemanager/webapp/RMWebServices.java | 123 +++++++ .../webapp/dao/AppGroupInfo.java | 79 ++++ .../webapp/dao/AppGroupRequest.java | 68 ++++ .../webapp/dao/AppGroupsInfo.java | 47 +++ ...yarn_server_resourcemanager_recovery.proto | 11 + .../resourcemanager/TestClientRMService.java | 161 +++++++-- .../recovery/TestZKRMStateStore.java | 83 +++++ .../DefaultClientRequestInterceptor.java | 25 ++ .../clientrm/FederationClientInterceptor.java | 25 ++ .../clientrm/RouterClientRMService.java | 28 ++ .../PassThroughClientRequestInterceptor.java | 25 ++ 67 files changed, 4375 insertions(+), 30 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/JettyUtils.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CreateApplicationGroupRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CreateApplicationGroupResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ListApplicationGroupsRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ListApplicationGroupsResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/TerminateApplicationGroupRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/TerminateApplicationGroupResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AppGroupACL.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationGroup.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationGroupState.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationGroupNotFoundException.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationGroupCLI.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CreateApplicationGroupRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CreateApplicationGroupResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ListApplicationGroupsRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ListApplicationGroupsResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TerminateApplicationGroupRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TerminateApplicationGroupResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationGroupPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AppGroupMonitor.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppGroupManager.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateRemoveAppGroupEvent.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreAppGroupEvent.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppGroupEvent.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationGroupStateData.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationGroupStateDataPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/group/RMAppGroup.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/group/RMAppGroupImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupsInfo.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/JettyUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/JettyUtils.java new file mode 100644 index 00000000000..29c0930968e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/JettyUtils.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.http; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Contains utility methods and constants relating to Jetty. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class JettyUtils { + public static final String UTF_8 = "charset=utf-8"; + public static final int HEADER_SIZE = 1024 * 64; + + private JettyUtils() { + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 62aa4972929..5db8430f79a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -517,4 +518,22 @@ public void killApplication(ApplicationId appId, String diagnostics) throws YarnException, IOException { client.killApplication(appId, diagnostics); } + + @Override + public void createApplicationGroup(String groupName, String permission, + boolean persistent) throws YarnException, IOException{ + client.createApplicationGroup(groupName, permission, persistent); + } + + @Override + public void terminateApplicationGroup(String groupName) + throws YarnException, IOException{ + client.terminateApplicationGroup(groupName); + } + + @Override + public List listApplicationGroups() + throws YarnException, IOException{ + return client.listApplicationGroups(); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index 65eac654845..daaf4a51037 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -70,6 +70,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; @@ -106,6 +108,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; @@ -122,6 +126,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -494,6 +500,27 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( throws YarnException, IOException { return null; } + + @Override + public CreateApplicationGroupResponse createApplicationGroup( + CreateApplicationGroupRequest request) + throws YarnException, IOException { + return null; + } + + @Override + public TerminateApplicationGroupResponse terminateApplicationGroup( + TerminateApplicationGroupRequest request) + throws YarnException, IOException { + return null; + } + + @Override + public ListApplicationGroupsResponse listApplicationGroups( + ListApplicationGroupsRequest request) + throws YarnException, IOException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 708a7c55976..a6b32162645 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -154,7 +154,7 @@ if "%1" == "--loglevel" ( ) set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^ - application applicationattempt cluster container node queue logs daemonlog historyserver ^ + application applicationattempt applicationgroup cluster container node queue logs daemonlog historyserver ^ timelineserver router timelinereader classpath for %%i in ( %yarncommands% ) do ( if %yarn-command% == %%i set yarncommand=true @@ -196,6 +196,12 @@ goto :eof set yarn-command-arguments=%yarn-command% %yarn-command-arguments% goto :eof +:applicationgroup + set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationGroupCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + set yarn-command-arguments=%yarn-command% %yarn-command-arguments% + goto :eof + :cluster set CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index 6d39366dccd..fba6723a5c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -19,11 +19,14 @@ package org.apache.hadoop.yarn.api; import java.io.IOException; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.io.retry.Idempotent; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -47,6 +50,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; @@ -57,6 +62,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -66,6 +73,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -589,4 +597,52 @@ SignalContainerResponse signalToContainer( public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( UpdateApplicationTimeoutsRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by client to create application group in the cluster + *

+ * + * @param request to create application group in this cluster + * @return an empty response + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public CreateApplicationGroupResponse createApplicationGroup( + CreateApplicationGroupRequest request) + throws YarnException, IOException; + + /** + *

+ * The interface used by client to terminate application group in the cluster + *

+ * + * @param request to terminate application group in this cluster + * @return an empty response + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public TerminateApplicationGroupResponse terminateApplicationGroup( + TerminateApplicationGroupRequest request) + throws YarnException, IOException; + + /** + *

+ * The interface used by client to get application group in the cluster + *

+ * + * @param request to get application group in this cluster + * @return an list of application groups in this cluster + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public ListApplicationGroupsResponse listApplicationGroups( + ListApplicationGroupsRequest request) + throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CreateApplicationGroupRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CreateApplicationGroupRequest.java new file mode 100644 index 00000000000..14d17ac6509 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CreateApplicationGroupRequest.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.EnumSet; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The request from clients to create application group.

+ * + */ +@Public +@Stable +public abstract class CreateApplicationGroupRequest { + @Public + @Stable + public static CreateApplicationGroupRequest newInstance(String groupName, + String permission, boolean persistent) { + CreateApplicationGroupRequest request = + Records.newRecord(CreateApplicationGroupRequest.class); + request.setGroupName(groupName); + request.setPermission(permission); + request.setIfPersistent(persistent); + return request; + } + + @Public + @Stable + public static CreateApplicationGroupRequest newInstance() { + CreateApplicationGroupRequest request = + Records.newRecord(CreateApplicationGroupRequest.class); + return request; + } + + /** + * get the group name + */ + public abstract String getGroupName(); + + /** + * Set the group name in request + */ + public abstract void setGroupName(String groupName); + + /** + * get the group permission + */ + public abstract String getPermission(); + + /** + * Set the group permission + */ + public abstract void setPermission(String permission); + + /** + * get if is a persistent group + */ + public abstract boolean isPersistent(); + + /** + * Set if is a persistent group + */ + public abstract void setIfPersistent(boolean persistent); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CreateApplicationGroupResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CreateApplicationGroupResponse.java new file mode 100644 index 00000000000..52e9d3e1a00 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/CreateApplicationGroupResponse.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The response sent by the ResourceManager to a client + * requesting create application group.

+ *

Currently, this is empty.

+ * + */ +@Public +@Stable +public abstract class CreateApplicationGroupResponse { + + @Public + @Stable + public static CreateApplicationGroupResponse + newInstance() { + CreateApplicationGroupResponse response = + Records.newRecord(CreateApplicationGroupResponse.class); + return response; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ListApplicationGroupsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ListApplicationGroupsRequest.java new file mode 100644 index 00000000000..ccd53ef0839 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ListApplicationGroupsRequest.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.EnumSet; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The request from clients to get list of application group + * in the cluster from the ResourceManager.

+ * + */ +@Public +@Stable +public abstract class ListApplicationGroupsRequest { + @Public + @Stable + public static ListApplicationGroupsRequest newInstance() { + ListApplicationGroupsRequest request = + Records.newRecord(ListApplicationGroupsRequest.class); + return request; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ListApplicationGroupsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ListApplicationGroupsResponse.java new file mode 100644 index 00000000000..975fde28d4d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ListApplicationGroupsResponse.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The response sent by the ResourceManager to a client + * requesting a application group list.

+ * + */ +@Public +@Stable +public abstract class ListApplicationGroupsResponse { + + @Private + @Unstable + public static ListApplicationGroupsResponse + newInstance(List applicationGroupList) { + ListApplicationGroupsResponse response = + Records.newRecord(ListApplicationGroupsResponse.class); + response.setApplicationGroupList(applicationGroupList); + return response; + } + + /** + * Get application group list for all nodes in the cluster. + * @return ApplicationGroup for all groups in the cluster + */ + @Public + @Stable + public abstract List getApplicationGroupList(); + + @Private + @Unstable + public abstract void setApplicationGroupList( + List applicationGroupList); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/TerminateApplicationGroupRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/TerminateApplicationGroupRequest.java new file mode 100644 index 00000000000..371465a23db --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/TerminateApplicationGroupRequest.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.EnumSet; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The request from clients to terminate application group + * in the cluster from the ResourceManager.

+ * + */ +@Public +@Stable +public abstract class TerminateApplicationGroupRequest { + @Public + @Stable + public static TerminateApplicationGroupRequest newInstance(String groupName) { + TerminateApplicationGroupRequest request = + Records.newRecord(TerminateApplicationGroupRequest.class); + request.setGroupName(groupName); + return request; + } + + @Public + @Stable + public static TerminateApplicationGroupRequest newInstance() { + TerminateApplicationGroupRequest request = + Records.newRecord(TerminateApplicationGroupRequest.class); + return request; + } + + public abstract String getGroupName(); + + public abstract void setGroupName(String groupName); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/TerminateApplicationGroupResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/TerminateApplicationGroupResponse.java new file mode 100644 index 00000000000..f0f0ccfeb59 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/TerminateApplicationGroupResponse.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The response sent by the ResourceManager to a client + * terminating application group.

+ * + */ +@Public +@Stable +public abstract class TerminateApplicationGroupResponse { + + @Private + @Unstable + public static TerminateApplicationGroupResponse newInstance() { + TerminateApplicationGroupResponse response = + Records.newRecord(TerminateApplicationGroupResponse.class); + return response; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AppGroupACL.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AppGroupACL.java new file mode 100644 index 00000000000..27e9290bec1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AppGroupACL.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + * {@code AppGroupACL} enumerates the various ACLs for application groups. + *

+ * The ACL is one of: + *

    + *
  • + * {@link #JOIN_GROUP} - ACL to let applications to join the group. + *
  • + *
  • {@link #ADMINISTER_GROUP} - ACL to administer the group.
  • + *
+ * + */ +@Public +@Stable +public enum AppGroupACL { + /** + * ACL to let applications to join the group. + */ + JOIN_GROUP, + + /** + * ACL to administer the group. + */ + ADMINISTER_GROUP, +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationGroup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationGroup.java new file mode 100644 index 00000000000..e1d2daf3d98 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationGroup.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationGroupStateProto; +import org.apache.hadoop.yarn.util.Records; + +/** + * {@code ApplicationGroup} is a metadata of an ApplicationGroup. + *

+ * It includes details such as: + *

    + *
  • Application group name.
  • + *
  • Group permission
  • + *
  • if persistent
  • + *
+ */ + +@Public +public abstract class ApplicationGroup { + + public static ApplicationGroup newInstance(String groupName, String permission, boolean persistent) { + ApplicationGroup group = Records.newRecord(ApplicationGroup.class); + group.setGroupName(groupName); + group.setPermission(permission); + group.setIsPersistent(persistent); + group.build(); + return group; + } + + public static ApplicationGroup newInstance(String groupName, String permission, + boolean persistent, YarnApplicationGroupState state) { + ApplicationGroup group = Records.newRecord(ApplicationGroup.class); + group.setGroupName(groupName); + group.setPermission(permission); + group.setIsPersistent(persistent); + group.setStatus(state); + group.build(); + return group; + } + + /** + * Get the group name. + * + * @return group name. + */ + public abstract String getGroupName(); + + /** + * Set the group name. + * + * @return void + */ + public abstract void setGroupName(String groupName); + + /** + * Get the group permission. + * + * @return group permission. + */ + public abstract String getPermission(); + + /** + * Set the permission. + * + * @return void + */ + public abstract void setPermission(String permission); + + /** + * Get the group permission. + * + * @return group permission. + */ + public abstract boolean getIsPersistent(); + + /** + * Set the persistent. + * + * @return void + */ + public abstract void setIsPersistent(boolean persistent); + + /** + * Get the group status. + * + * @return group status. + */ + public abstract YarnApplicationGroupState getStatus(); + + /** + * Set the status. + * + * @return void + */ + public abstract void setStatus(YarnApplicationGroupState status); + + /** + * Convert to proto struct + * + */ + protected abstract void build(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationGroupState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationGroupState.java new file mode 100644 index 00000000000..ebfc907cb02 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationGroupState.java @@ -0,0 +1,33 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + * Enumeration of various states of an Application Group. + */ +@Public +@Stable +public enum YarnApplicationGroupState { + ACTIVE, + TERMINATING, + TERMINATED, +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 2a8185eb039..ca215758e7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3259,6 +3259,19 @@ public static boolean areNodeLabelsEnabled( public static final String TIMELINE_XFS_OPTIONS = TIMELINE_XFS_PREFIX + "xframe-options"; + public static final String RM_APPLICATION_GROUP_ENABLED = + RM_PREFIX + "application-group.enabled"; + public static final boolean DEFAULT_RM_APPLICATION_GROUP_ENABLED = + false; + + /** + * Specifies the time(seconds) to remove terminated app group. + */ + public static final String RM_APPLICATION_GROUP_TERMINATED_EXPIRE_TIME = + RM_PREFIX + "application-group.terminated.expire-time"; + public static final long DEFAULT_RM_APPLICATION_GROUP_TERMINATED_EXPIRE_TIME = + 86400; + public YarnConfiguration() { super(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationGroupNotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationGroupNotFoundException.java new file mode 100644 index 00000000000..9deb5032a61 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationGroupNotFoundException.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.exceptions; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +@Public +@Unstable +public class ApplicationGroupNotFoundException extends YarnException{ + + private static final long serialVersionUID = 8694409L; + + public ApplicationGroupNotFoundException(Throwable cause) { + super(cause); + } + + public ApplicationGroupNotFoundException(String message) { + super(message); + } + + public ApplicationGroupNotFoundException(String message, + Throwable cause) { + super(message, cause); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index ba79db09a6f..38b5f4022e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -61,4 +61,7 @@ service ApplicationClientProtocolService { rpc updateApplicationPriority (UpdateApplicationPriorityRequestProto) returns (UpdateApplicationPriorityResponseProto); rpc signalToContainer(SignalContainerRequestProto) returns (SignalContainerResponseProto); rpc updateApplicationTimeouts (UpdateApplicationTimeoutsRequestProto) returns (UpdateApplicationTimeoutsResponseProto); + rpc createApplicationGroup (CreateApplicationGroupRequestProto) returns (CreateApplicationGroupResponseProto); + rpc terminateApplicationGroup (TerminateApplicationGroupRequestProto) returns (TerminateApplicationGroupResponseProto); + rpc listApplicationGroups (ListApplicationGroupsRequestProto) returns (ListApplicationGroupsResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 80c043fbab9..caa821acb4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -142,6 +142,19 @@ message ContainerReportProto { optional ExecutionTypeProto executionType = 12 [default = GUARANTEED]; } +enum ApplicationGroupStateProto { + ACTIVE = 1; + TERMINATING = 2; + TERMINATED = 3; +} + +message ApplicationGroupProto { + optional string group_name = 1; + optional string permission = 2; + optional bool persistent = 3; + optional ApplicationGroupStateProto status = 4; +} + enum YarnApplicationStateProto { NEW = 1; NEW_SAVING = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 8301971f6d2..11304a56095 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -494,3 +494,26 @@ message RunSharedCacheCleanerTaskRequestProto { message RunSharedCacheCleanerTaskResponseProto { optional bool accepted = 1; } + +message CreateApplicationGroupRequestProto { + optional string groupName = 1; + optional string permission = 2; + optional bool persistent = 3; +} + +message CreateApplicationGroupResponseProto { +} + +message TerminateApplicationGroupRequestProto { + optional string groupName = 1; +} + +message TerminateApplicationGroupResponseProto { +} + +message ListApplicationGroupsRequestProto { +} + +message ListApplicationGroupsResponseProto { + repeated ApplicationGroupProto applicationGroups = 1; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 8c68a31ffe2..ca5385c8c99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -855,4 +856,41 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( throw new UnsupportedOperationException("The sub-class extending " + YarnClient.class.getName() + " is expected to implement this !"); } + + /** + *

+ * Create an application group. + *

+ * + * @param groupName group name + * @param permission group join and terminate permission + * @param persistent group if persistent + * @throws YarnException + * @throws IOException + */ + public abstract void createApplicationGroup(String groupName, String permission, + boolean persistent) throws YarnException, IOException; + + /** + *

+ * Terminate an application group. + *

+ * + * @param groupName group name + * @throws YarnException + * @throws IOException + */ + public abstract void terminateApplicationGroup(String groupName) throws YarnException, IOException; + + /** + * + *

+ * Get all the application groups status + *

+ * + * @return list of application groups status + * @throws YarnException + * @throws IOException + */ + public abstract List listApplicationGroups() throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 83210bd28a5..01bb7582c52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -42,6 +42,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; @@ -72,6 +73,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; @@ -83,11 +85,13 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse; import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -937,4 +941,28 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( throws YarnException, IOException { return rmClient.updateApplicationTimeouts(request); } + + @Override + public void createApplicationGroup(String groupName, String permission, + boolean persistent) throws YarnException, IOException{ + CreateApplicationGroupRequest request = + CreateApplicationGroupRequest.newInstance(groupName, permission, persistent); + rmClient.createApplicationGroup(request); + } + + @Override + public void terminateApplicationGroup(String groupName) + throws YarnException, IOException{ + TerminateApplicationGroupRequest request = + TerminateApplicationGroupRequest.newInstance(groupName); + rmClient.terminateApplicationGroup(request); + } + + @Override + public List listApplicationGroups() + throws YarnException, IOException{ + ListApplicationGroupsRequest request = + ListApplicationGroupsRequest.newInstance(); + return rmClient.listApplicationGroups(request).getApplicationGroupList(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationGroupCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationGroupCLI.java new file mode 100644 index 00000000000..98a67b7e503 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationGroupCLI.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.client.cli; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.Charset; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Options; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; +import org.apache.hadoop.yarn.exceptions.ApplicationGroupNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; + +@Private +@Unstable +public class ApplicationGroupCLI extends YarnCLI { + + private static final String CREATE_CMD = "create"; + private static final String TERMINATE_CMD = "terminate"; + private static final String PERMISSION = "permission"; + private static final String PERSISTENT = "persistent"; + + public static final String APPLICATIONGROUP = "applicationGroup"; + + private static final String APPLICATIONS_GROUP_PATTERN = + "%30s\t%20s\t%30s\t%20s" + + System.getProperty("line.separator"); + + public static void main(String[] args) throws Exception { + ApplicationGroupCLI cli = new ApplicationGroupCLI(); + cli.setSysOutPrintStream(System.out); + cli.setSysErrPrintStream(System.err); + int res = ToolRunner.run(cli, args); + cli.stop(); + System.exit(res); + } + + @Override + public int run(String[] args) throws Exception { + Options opts = new Options(); + String title = null; + if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATIONGROUP)) { + title = APPLICATIONGROUP; + opts.addOption(CREATE_CMD, true, + "Create the application group."); + opts.addOption(PERMISSION, true, "Application group join and terminate permission."); + opts.addOption(PERSISTENT, false, "Create a persistent application group."); + opts.addOption(TERMINATE_CMD, true, "Terminate a application group."); + opts.addOption(LIST_CMD, false, "List application groups."); + } + + int exitCode = -1; + CommandLine cliParser = null; + try { + cliParser = new GnuParser().parse(opts, args); + } catch (MissingArgumentException ex) { + sysout.println("Missing argument for options"); + printUsage(title, opts); + return exitCode; + } + + if (cliParser.hasOption(CREATE_CMD)) { + if (args[0].equalsIgnoreCase(APPLICATIONGROUP)) { + String groupName = cliParser.getOptionValue(CREATE_CMD); + String permission = ""; + boolean persistent = false; + + if(cliParser.hasOption(PERSISTENT)){ + persistent = true; + } + + if(cliParser.hasOption(PERMISSION)){ + permission = cliParser.getOptionValue(PERMISSION); + } + + exitCode = createApplicationGroup(groupName, permission, persistent); + } + return exitCode; + } else if (cliParser.hasOption(LIST_CMD)) { + if (args[0].equalsIgnoreCase(APPLICATIONGROUP)) { + listApplicationGroups(); + } + + } else if (cliParser.hasOption(TERMINATE_CMD)) { + if (args[0].equalsIgnoreCase(APPLICATIONGROUP)) { + String groupName = cliParser.getOptionValue(TERMINATE_CMD); + exitCode = terminateApplicationGroup(groupName); + } + return exitCode; + } else { + syserr.println("Invalid Command Usage : "); + printUsage(title, opts); + } + return 0; + } + + private int createApplicationGroup(String groupName, String permission, + boolean persistent) throws YarnException, IOException { + int returnCode = -1; + try { + client.createApplicationGroup(groupName, permission, persistent); + returnCode = 0; + } catch (Exception exception) { + sysout.println("Create operation failed with " + exception.getMessage()); + } + return returnCode; + } + + private int terminateApplicationGroup(String groupName) throws YarnException, + IOException { + int returnCode = -1; + try { + client.terminateApplicationGroup(groupName); + returnCode = 0; + } catch (ApplicationGroupNotFoundException exception) { + + sysout.println("Terminate operation failed with " + exception.getMessage()); + } + return returnCode; + } + + private void listApplicationGroups() throws YarnException, + IOException { + + PrintWriter writer = new PrintWriter( + new OutputStreamWriter(sysout, Charset.forName("UTF-8"))); + + List appsGroups = client.listApplicationGroups(); + + writer.println("Total number of groups " + ":" + appsGroups.size()); + writer.printf(APPLICATIONS_GROUP_PATTERN, "GroupName", "Status", + "Permission", "Persistent"); + for (ApplicationGroup group : appsGroups) { + writer.printf( + APPLICATIONS_GROUP_PATTERN, + group.getGroupName(), + group.getStatus().name(), + group.getPermission(), + Boolean.toString(group.getIsPersistent())); + } + writer.flush(); + } + + private void printUsage(String title, Options opts) { + new HelpFormatter().printHelp(title, opts); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index ad7cb296080..506a9ddad6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -21,6 +21,7 @@ import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; @@ -33,6 +34,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; @@ -69,6 +72,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; @@ -81,6 +86,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -91,6 +98,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CreateApplicationGroupRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CreateApplicationGroupResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FailApplicationAttemptRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FailApplicationAttemptResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptReportRequestPBImpl; @@ -127,6 +136,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueUserAclsInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.KillApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.KillApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ListApplicationGroupsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ListApplicationGroupsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.MoveApplicationAcrossQueuesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.MoveApplicationAcrossQueuesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl; @@ -139,6 +150,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationSubmissionResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.TerminateApplicationGroupRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.TerminateApplicationGroupResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UpdateApplicationPriorityRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UpdateApplicationPriorityResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UpdateApplicationTimeoutsRequestPBImpl; @@ -147,9 +160,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.CreateApplicationGroupRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FailApplicationAttemptRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsRequestProto; @@ -164,6 +179,7 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ListApplicationGroupsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.MoveApplicationAcrossQueuesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionRequestProto; @@ -171,6 +187,7 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateApplicationPriorityRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateApplicationTimeoutsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.TerminateApplicationGroupRequestProto; import com.google.protobuf.ServiceException; @@ -619,4 +636,47 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( return null; } } + + @Override + public CreateApplicationGroupResponse createApplicationGroup( + CreateApplicationGroupRequest request) throws YarnException, IOException { + CreateApplicationGroupRequestProto requestProto = + ((CreateApplicationGroupRequestPBImpl) request).getProto(); + try { + return new CreateApplicationGroupResponsePBImpl( + proxy.createApplicationGroup(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public TerminateApplicationGroupResponse terminateApplicationGroup( + TerminateApplicationGroupRequest request) + throws YarnException, IOException { + TerminateApplicationGroupRequestProto requestProto = + ((TerminateApplicationGroupRequestPBImpl) request).getProto(); + try { + return new TerminateApplicationGroupResponsePBImpl( + proxy.terminateApplicationGroup(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public ListApplicationGroupsResponse listApplicationGroups( + ListApplicationGroupsRequest request) throws YarnException, IOException { + YarnServiceProtos.ListApplicationGroupsRequestProto requestProto = + ((ListApplicationGroupsRequestPBImpl) request).getProto(); + try { + return new ListApplicationGroupsResponsePBImpl( + proxy.listApplicationGroups(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index 93ce6a343c5..d0d13d47a52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; @@ -48,18 +49,22 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CreateApplicationGroupRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CreateApplicationGroupResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FailApplicationAttemptRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FailApplicationAttemptResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptReportRequestPBImpl; @@ -96,6 +101,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueUserAclsInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.KillApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.KillApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ListApplicationGroupsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ListApplicationGroupsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.MoveApplicationAcrossQueuesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.MoveApplicationAcrossQueuesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl; @@ -110,6 +117,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.TerminateApplicationGroupRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.TerminateApplicationGroupResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UpdateApplicationPriorityRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UpdateApplicationPriorityResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UpdateApplicationTimeoutsRequestPBImpl; @@ -118,6 +127,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.CreateApplicationGroupRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.CreateApplicationGroupResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FailApplicationAttemptRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FailApplicationAttemptResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto; @@ -152,6 +163,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ListApplicationGroupsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ListApplicationGroupsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.MoveApplicationAcrossQueuesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.MoveApplicationAcrossQueuesResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationDeleteRequestProto; @@ -169,6 +182,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateApplicationTimeoutsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.TerminateApplicationGroupRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.TerminateApplicationGroupResponseProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -631,4 +646,56 @@ public UpdateApplicationTimeoutsResponseProto updateApplicationTimeouts( throw new ServiceException(e); } } + + @Override + public CreateApplicationGroupResponseProto createApplicationGroup( + RpcController controller, CreateApplicationGroupRequestProto proto) + throws ServiceException { + CreateApplicationGroupRequestPBImpl request = + new CreateApplicationGroupRequestPBImpl(proto); + try { + CreateApplicationGroupResponse response = + real.createApplicationGroup(request); + return ((CreateApplicationGroupResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public TerminateApplicationGroupResponseProto terminateApplicationGroup( + RpcController controller, TerminateApplicationGroupRequestProto proto) + throws ServiceException { + TerminateApplicationGroupRequestPBImpl request = + new TerminateApplicationGroupRequestPBImpl(proto); + try { + TerminateApplicationGroupResponse response = + real.terminateApplicationGroup(request); + return ((TerminateApplicationGroupResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public ListApplicationGroupsResponseProto listApplicationGroups( + RpcController controller, + ListApplicationGroupsRequestProto proto) + throws ServiceException { + ListApplicationGroupsRequestPBImpl request = + new ListApplicationGroupsRequestPBImpl(proto); + try { + ListApplicationGroupsResponse response = + real.listApplicationGroups(request); + return ((ListApplicationGroupsResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CreateApplicationGroupRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CreateApplicationGroupRequestPBImpl.java new file mode 100644 index 00000000000..d9f28c55c70 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CreateApplicationGroupRequestPBImpl.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.CreateApplicationGroupRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.CreateApplicationGroupRequestProtoOrBuilder; + +import com.google.protobuf.TextFormat; + +/** + *

An implementation of CreateApplicationGroupRequest.

+ * + * @see CreateApplicationGroupRequest + */ + +@Private +@Unstable +public class CreateApplicationGroupRequestPBImpl extends CreateApplicationGroupRequest { + CreateApplicationGroupRequestProto proto = CreateApplicationGroupRequestProto.getDefaultInstance(); + CreateApplicationGroupRequestProto.Builder builder = null; + boolean viaProto = false; + String groupName = null; + String permission = null; + + public CreateApplicationGroupRequestPBImpl() { + builder = CreateApplicationGroupRequestProto.newBuilder(); + } + + public CreateApplicationGroupRequestPBImpl(CreateApplicationGroupRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized CreateApplicationGroupRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToBuilder() { + if (groupName != null) { + builder.setGroupName(groupName); + } + if (permission != null) { + builder.setPermission(permission); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = CreateApplicationGroupRequestProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String getGroupName() { + if (this.groupName != null) { + return this.groupName; + } + + CreateApplicationGroupRequestProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasGroupName()) { + return null; + } + this.groupName = p.getGroupName(); + return this.groupName; + } + + @Override + public void setGroupName(String name) { + maybeInitBuilder(); + if (this.groupName == null) { + builder.clearGroupName(); + } + this.groupName = name; + } + + @Override + public String getPermission() { + if (this.permission != null) { + return this.permission; + } + + CreateApplicationGroupRequestProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasPermission()) { + return null; + } + this.permission = p.getPermission(); + return this.permission; + } + + @Override + public void setPermission(String permission) { + maybeInitBuilder(); + if (this.permission == null) { + builder.clearPermission(); + } + this.permission = permission; + } + + @Override + public boolean isPersistent() { + CreateApplicationGroupRequestProtoOrBuilder p = viaProto ? proto : builder; + return p.getPersistent(); + } + + @Override + public void setIfPersistent(boolean persistent) { + maybeInitBuilder(); + builder.setPersistent(persistent); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CreateApplicationGroupResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CreateApplicationGroupResponsePBImpl.java new file mode 100644 index 00000000000..12fea390750 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/CreateApplicationGroupResponsePBImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.CreateApplicationGroupResponseProto; + +import com.google.protobuf.TextFormat; + +/** + *

An implementation of CreateApplicationGroupResponse.

+ * + * @see CreateApplicationGroupResponse + */ + +@Private +@Unstable +public class CreateApplicationGroupResponsePBImpl extends CreateApplicationGroupResponse { + CreateApplicationGroupResponseProto proto = CreateApplicationGroupResponseProto.getDefaultInstance(); + CreateApplicationGroupResponseProto.Builder builder = null; + boolean viaProto = false; + + public CreateApplicationGroupResponsePBImpl() { + builder = CreateApplicationGroupResponseProto.newBuilder(); + } + + public CreateApplicationGroupResponsePBImpl(CreateApplicationGroupResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public CreateApplicationGroupResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ListApplicationGroupsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ListApplicationGroupsRequestPBImpl.java new file mode 100644 index 00000000000..9ffb76a5ddf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ListApplicationGroupsRequestPBImpl.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ListApplicationGroupsRequestProto; + +import com.google.protobuf.TextFormat; + +/** + *

An implementation of ListApplicationGroupsRequest.

+ * + * @see ListApplicationGroupsRequest + */ + +@Private +@Unstable +public class ListApplicationGroupsRequestPBImpl extends ListApplicationGroupsRequest { + ListApplicationGroupsRequestProto proto = ListApplicationGroupsRequestProto.getDefaultInstance(); + ListApplicationGroupsRequestProto.Builder builder = null; + boolean viaProto = false; + + public ListApplicationGroupsRequestPBImpl() { + builder = ListApplicationGroupsRequestProto.newBuilder(); + } + + public ListApplicationGroupsRequestPBImpl(ListApplicationGroupsRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized ListApplicationGroupsRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ListApplicationGroupsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ListApplicationGroupsResponsePBImpl.java new file mode 100644 index 00000000000..a02e7a4879e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ListApplicationGroupsResponsePBImpl.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationGroupPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationGroupProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ListApplicationGroupsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ListApplicationGroupsResponseProtoOrBuilder; +import com.google.protobuf.TextFormat; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + *

An implementation of ListApplicationGroupsResponse.

+ * + * @see ListApplicationGroupsResponse + */ + +@Private +@Unstable +public class ListApplicationGroupsResponsePBImpl extends ListApplicationGroupsResponse { + ListApplicationGroupsResponseProto proto = ListApplicationGroupsResponseProto.getDefaultInstance(); + ListApplicationGroupsResponseProto.Builder builder = null; + boolean viaProto = false; + + List applicationGroupList; + + public ListApplicationGroupsResponsePBImpl() { + builder = ListApplicationGroupsResponseProto.newBuilder(); + } + + public ListApplicationGroupsResponsePBImpl(ListApplicationGroupsResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public ListApplicationGroupsResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.applicationGroupList != null) { + addLocalNodeManagerInfosToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ListApplicationGroupsResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public List getApplicationGroupList() { + initLocalNodeManagerInfosList(); + return this.applicationGroupList; + } + + @Override + public void setApplicationGroupList( + List applicationGroupList) { + maybeInitBuilder(); + if (applicationGroupList == null) + builder.clearApplicationGroups(); + this.applicationGroupList = applicationGroupList; + } + + // Once this is called. containerList will never be null - until a getProto + // is called. + private void initLocalNodeManagerInfosList() { + if (this.applicationGroupList != null) { + return; + } + ListApplicationGroupsResponseProtoOrBuilder + p = viaProto ? proto : builder; + List list = p.getApplicationGroupsList(); + applicationGroupList = new ArrayList(); + + for (ApplicationGroupProto a : list) { + applicationGroupList.add(convertFromProtoFormat(a)); + } + } + + private void addLocalNodeManagerInfosToProto() { + maybeInitBuilder(); + builder.clearApplicationGroups(); + if (applicationGroupList == null) + return; + Iterable iterable = new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = applicationGroupList.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ApplicationGroupProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllApplicationGroups(iterable); + } + + private ApplicationGroupPBImpl convertFromProtoFormat(ApplicationGroupProto p) { + return new ApplicationGroupPBImpl(p); + } + + private ApplicationGroupProto convertToProtoFormat(ApplicationGroup t) { + return ((ApplicationGroupPBImpl)t).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TerminateApplicationGroupRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TerminateApplicationGroupRequestPBImpl.java new file mode 100644 index 00000000000..81fb2c8cd35 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TerminateApplicationGroupRequestPBImpl.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.TerminateApplicationGroupRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.TerminateApplicationGroupRequestProtoOrBuilder; + +import com.google.protobuf.TextFormat; + +/** + *

An implementation of TerminateApplicationGroupRequest.

+ * + * @see TerminateApplicationGroupRequest + */ + +@Private +@Unstable +public class TerminateApplicationGroupRequestPBImpl extends TerminateApplicationGroupRequest { + TerminateApplicationGroupRequestProto proto = TerminateApplicationGroupRequestProto.getDefaultInstance(); + TerminateApplicationGroupRequestProto.Builder builder = null; + boolean viaProto = false; + String groupName = null; + String permission = null; + + public TerminateApplicationGroupRequestPBImpl() { + builder = TerminateApplicationGroupRequestProto.newBuilder(); + } + + public TerminateApplicationGroupRequestPBImpl(TerminateApplicationGroupRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized TerminateApplicationGroupRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToBuilder() { + if (groupName != null) { + builder.setGroupName(groupName); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = TerminateApplicationGroupRequestProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String getGroupName() { + if (this.groupName != null) { + return this.groupName; + } + + TerminateApplicationGroupRequestProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasGroupName()) { + return null; + } + this.groupName = p.getGroupName(); + return this.groupName; + } + + @Override + public void setGroupName(String name) { + maybeInitBuilder(); + if (this.groupName == null) { + builder.clearGroupName(); + } + this.groupName = name; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TerminateApplicationGroupResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TerminateApplicationGroupResponsePBImpl.java new file mode 100644 index 00000000000..afe900f507c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TerminateApplicationGroupResponsePBImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.TerminateApplicationGroupResponseProto; + +import com.google.protobuf.TextFormat; + +/** + *

An implementation of TerminateApplicationGroupResponse.

+ * + * @see TerminateApplicationGroupResponse + */ + +@Private +@Unstable +public class TerminateApplicationGroupResponsePBImpl extends TerminateApplicationGroupResponse { + TerminateApplicationGroupResponseProto proto = TerminateApplicationGroupResponseProto.getDefaultInstance(); + TerminateApplicationGroupResponseProto.Builder builder = null; + boolean viaProto = false; + + public TerminateApplicationGroupResponsePBImpl() { + builder = TerminateApplicationGroupResponseProto.newBuilder(); + } + + public TerminateApplicationGroupResponsePBImpl(TerminateApplicationGroupResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public TerminateApplicationGroupResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationGroupPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationGroupPBImpl.java new file mode 100644 index 00000000000..39abf272c39 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationGroupPBImpl.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationGroupProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationGroupStateProto; + +import com.google.common.base.Preconditions; + +/** + *

An implementation of ApplicationGroup.

+ * + * @see ApplicationGroup + */ + +@Private +@Unstable +public class ApplicationGroupPBImpl extends ApplicationGroup { + ApplicationGroupProto proto = null; + ApplicationGroupProto.Builder builder = null; + + public ApplicationGroupPBImpl() { + builder = ApplicationGroupProto.newBuilder(); + } + + public ApplicationGroupPBImpl(ApplicationGroupProto proto) { + this.proto = proto; + } + + public ApplicationGroupProto getProto() { + return proto; + } + + @Override + public String getGroupName() { + Preconditions.checkNotNull(proto); + return proto.getGroupName(); + } + + @Override + public void setGroupName(String groupName) { + Preconditions.checkNotNull(builder); + builder.setGroupName(groupName); + } + + @Override + public String getPermission() { + Preconditions.checkNotNull(proto); + return proto.getPermission(); + } + + @Override + public void setPermission(String permission) { + Preconditions.checkNotNull(builder); + builder.setPermission(permission); + } + + @Override + public boolean getIsPersistent() { + Preconditions.checkNotNull(proto); + return proto.getPersistent(); + } + + @Override + public void setIsPersistent(boolean persistent) { + Preconditions.checkNotNull(builder); + builder.setPersistent(persistent); + } + + @Override + public YarnApplicationGroupState getStatus() { + Preconditions.checkNotNull(proto); + return convertFromProtoFormat(proto.getStatus()); + } + + @Override + public void setStatus(YarnApplicationGroupState status) { + Preconditions.checkNotNull(builder); + builder.setStatus(convertToProtoFormat(status)); + } + + @Override + protected void build() { + proto = builder.build(); + builder = null; + } + + public static ApplicationGroupStateProto convertToProtoFormat(YarnApplicationGroupState e) { + return ApplicationGroupStateProto.valueOf(e.name()); + } + public static YarnApplicationGroupState convertFromProtoFormat(ApplicationGroupStateProto e) { + return YarnApplicationGroupState.valueOf(e.name()); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index 1e377690ba8..3914deab4a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; @@ -75,6 +77,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; @@ -93,6 +97,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -746,6 +752,34 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( return UpdateApplicationTimeoutsResponse.newInstance(); } + @Override + public CreateApplicationGroupResponse createApplicationGroup( + CreateApplicationGroupRequest request) throws YarnException, IOException { + + validateRunning(); + + return CreateApplicationGroupResponse.newInstance(); + } + + @Override + public TerminateApplicationGroupResponse terminateApplicationGroup( + TerminateApplicationGroupRequest request) + throws YarnException, IOException { + + validateRunning(); + + return TerminateApplicationGroupResponse.newInstance(); + } + + @Override + public ListApplicationGroupsResponse listApplicationGroups( + ListApplicationGroupsRequest request) throws YarnException, IOException { + + validateRunning(); + + return ListApplicationGroupsResponse.newInstance(null); + } + @Override public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) throws StandbyException, YarnException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AppGroupMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AppGroupMonitor.java new file mode 100644 index 00000000000..96b22a8135f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AppGroupMonitor.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppKillByClientEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.group.RMAppGroup; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + +import static org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.isFinalState; + +class AppGroupMonitor extends AbstractService { + + private static final Log LOG = LogFactory.getLog(AppGroupMonitor.class); + + protected final RMContext rmContext; + protected final RMAppGroupManager rmAppGroupManager; + private Thread monitorThread; + private long terminatedGroupExpiredTime = 24 * 60 * 60 * 1000; + private long timeToIgnoreNewGroup = 20 * 60 * 1000; + + public AppGroupMonitor(RMContext rmContext, + RMAppGroupManager rmAppGroupManager) { + super("AppGroupMonitor"); + this.rmContext = rmContext; + this.rmAppGroupManager = rmAppGroupManager; + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + terminatedGroupExpiredTime = 1000 * conf + .getLong(YarnConfiguration.RM_APPLICATION_GROUP_TERMINATED_EXPIRE_TIME, + YarnConfiguration.DEFAULT_RM_APPLICATION_GROUP_TERMINATED_EXPIRE_TIME); + } + + @Override + protected void serviceStart() throws Exception { + + monitorThread = new Thread(new Runnable() { + @Override + public void run() { + while (true) { + if (rmAppGroupManager.isRecovered()) { + ConcurrentMap rmApps = rmContext.getRMApps(); + ConcurrentMap rmAppGroups = + rmContext.getRMAppGroups(); + Set activeAppGroups = new HashSet(); + for (Map.Entry app : rmApps.entrySet()) { + String groupName = rmAppGroupManager + .getGroupFromTags(app.getValue().getApplicationTags()); + if (null != groupName) { + // app is running + if (!isFinalState(app.getValue().getState())) { + activeAppGroups.add(groupName); + + // kill the app as group already terminate + RMAppGroup appGroup = rmAppGroups.get(groupName); + if (null != appGroup) { + if (appGroup.getStatus() + == YarnApplicationGroupState.TERMINATING) { + + StringBuilder message = new StringBuilder(); + message.append("Application ").append(app.getKey()) + .append(" was killed as group terminated by user ") + .append(appGroup.getTerminateUser()); + + rmContext.getDispatcher().getEventHandler().handle( + new RMAppKillByClientEvent(app.getKey(), + message.toString(), null, null)); + } else if (appGroup.getStatus() + == YarnApplicationGroupState.TERMINATED) { + LOG.error( + "Unexpected error, group " + appGroup.getGroupName() + + " already terminated, application " + app + .getKey().toString() + "should not in " + app + .getValue().getState()); + } + } else { + LOG.error("Unexpected error, not find group " + appGroup + .getGroupName() + ", application " + app.getKey() + .toString() + "should not in this group."); + } + } + } + } + for (Map.Entry appGroup : rmAppGroups + .entrySet()) { + if (activeAppGroups.contains(appGroup.getKey())) { + appGroup.getValue().setHasRunningApplications(true); + } else { + appGroup.getValue().setHasRunningApplications(false); + } + } + + long currentTime = System.currentTimeMillis(); + + for (Map.Entry appGroup : rmAppGroups + .entrySet()) { + RMAppGroup groupStatus = appGroup.getValue(); + if (groupStatus.getStatus() + == YarnApplicationGroupState.TERMINATING && !groupStatus + .hasRunningApplications()) { + try { + rmAppGroupManager + .terminateApplicationGroup(groupStatus.getGroupName(), + null, true); + } catch (Exception e) { + LOG.warn("Fail to terminate application group with exception", + e); + } + } else if (groupStatus.getStatus() + == YarnApplicationGroupState.TERMINATED + && currentTime - groupStatus.getTerminateTime() + >= terminatedGroupExpiredTime) { + try { + rmAppGroupManager + .removeApplicationGroup(groupStatus.getGroupName(), true); + } catch (Exception e) { + LOG.warn("Fail to remove application group with exception", + e); + } + } + // terminated the non persistent group when no running applications + if (!groupStatus.isPersistent() && !groupStatus + .hasRunningApplications() + && currentTime - groupStatus.getCreateTime() + >= timeToIgnoreNewGroup) { + try { + rmAppGroupManager + .terminateApplicationGroup(groupStatus.getGroupName(), + "system", true); + } catch (Exception e) { + LOG.warn("Fail to terminate application group with exception", + e); + } + } + } + } + + try { + Thread.sleep(60000); + } catch (Exception e) { + // ignore + } + } + } + }); + + monitorThread.start(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (monitorThread != null) { + monitorThread.interrupt(); + } + super.serviceStop(); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 5b4123d1a64..6fbd43f7465 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; @@ -92,6 +94,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; @@ -108,6 +112,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -192,6 +198,7 @@ final private YarnScheduler scheduler; final private RMContext rmContext; private final RMAppManager rmAppManager; + private final RMAppGroupManager rmAppGroupManager; private Server server; protected RMDelegationTokenSecretManager rmDTSecretManager; @@ -220,10 +227,28 @@ public ClientRMService(RMContext rmContext, YarnScheduler scheduler, queueACLsManager, rmDTSecretManager, new UTCClock()); } + public ClientRMService(RMContext rmContext, YarnScheduler scheduler, + RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, + QueueACLsManager queueACLsManager, + RMDelegationTokenSecretManager rmDTSecretManager, + RMAppGroupManager rmAppGroupManager) { + this(rmContext, scheduler, rmAppManager, applicationACLsManager, + queueACLsManager, rmDTSecretManager, new UTCClock(), rmAppGroupManager); + } + public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, QueueACLsManager queueACLsManager, RMDelegationTokenSecretManager rmDTSecretManager, Clock clock) { + this(rmContext, scheduler, rmAppManager, applicationACLsManager, + queueACLsManager, rmDTSecretManager, clock, null); + } + + public ClientRMService(RMContext rmContext, YarnScheduler scheduler, + RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, + QueueACLsManager queueACLsManager, + RMDelegationTokenSecretManager rmDTSecretManager, Clock clock, + RMAppGroupManager rmAppGroupManager) { super(ClientRMService.class.getName()); this.scheduler = scheduler; this.rmContext = rmContext; @@ -234,6 +259,7 @@ public ClientRMService(RMContext rmContext, YarnScheduler scheduler, this.reservationSystem = rmContext.getReservationSystem(); this.clock = clock; this.rValidator = new ReservationInputValidator(clock); + this.rmAppGroupManager = rmAppGroupManager; } @Override @@ -625,6 +651,33 @@ public SubmitApplicationResponse submitApplication( } } + if (null != rmAppGroupManager) { + String groupName = rmAppGroupManager + .getGroupFromTags(submissionContext.getApplicationTags()); + + if (null != groupName) { + if (rmAppGroupManager.isApplicationGroupExist(groupName)) { + if (!rmAppGroupManager.checkAppGroupActive(groupName)) { + throw new YarnException( + "Group not in active status and can not join, groupName as " + + groupName); + } else if (!rmAppGroupManager + .checkJoinPermission(UserGroupInformation.getCurrentUser(), + groupName)) { + throw new YarnException( + "Application don't have permission to join the group " + + groupName); + } + } else { + createApplicationGroup( + CreateApplicationGroupRequest.newInstance(groupName, "", false)); + } + } + } else { + LOG.debug( + "rmAppGroupManager is null, maybe application group feature not enabled."); + } + ReservationId reservationId = request.getApplicationSubmissionContext() .getReservationID(); @@ -1726,6 +1779,91 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( return response; } + @Override + public CreateApplicationGroupResponse createApplicationGroup( + CreateApplicationGroupRequest request) throws YarnException, IOException { + + String groupName = request.getGroupName(); + String permission = request.getPermission(); + boolean isPersistent = request.isPersistent(); + UserGroupInformation callerUGI; + + try { + callerUGI = UserGroupInformation.getCurrentUser(); + } catch (IOException ie) { + LOG.info("Error getting UGI ", ie); + RMAuditLogger.logFailure("UNKNOWN", + AuditConstants.CREATE_APPLICATION_GROUP_REQUEST, "UNKNOWN", + "ClientRMService", "Error getting UGI"); + throw RPCUtil.getRemoteException(ie); + } + + if (rmContext.getRMAppGroups().containsKey(groupName)) { + throw new YarnException("Group " + groupName + " already exist."); + } + + if(permission.isEmpty()){ + permission = + "joinACL:" + callerUGI.getShortUserName() + ";adminACL:" + callerUGI + .getShortUserName(); + } + + rmAppGroupManager.createApplicationGroup(groupName, permission, isPersistent, + callerUGI.getShortUserName(), false); + + CreateApplicationGroupResponse response = recordFactory + .newRecordInstance(CreateApplicationGroupResponse.class); + return response; + } + + @Override + public TerminateApplicationGroupResponse terminateApplicationGroup( + TerminateApplicationGroupRequest request) + throws YarnException, IOException { + + String groupName = request.getGroupName(); + UserGroupInformation callerUGI; + + try { + callerUGI = UserGroupInformation.getCurrentUser(); + } catch (IOException ie) { + LOG.info("Error getting UGI ", ie); + RMAuditLogger.logFailure("UNKNOWN", + AuditConstants.CREATE_APPLICATION_GROUP_REQUEST, "UNKNOWN", + "ClientRMService", "Error getting UGI"); + throw RPCUtil.getRemoteException(ie); + } + + if (!rmAppGroupManager.checkTerminatePermisson(callerUGI, groupName)) { + RMAuditLogger.logFailure(callerUGI.getShortUserName(), "Terminate Application Group", + "User doesn't have permissions to terminate application group " + + groupName, + "ClientRMService", AuditConstants.UNAUTHORIZED_USER); + throw RPCUtil.getRemoteException(new AccessControlException("User " + + callerUGI.getShortUserName() + " cannot terminate " + + groupName)); + } + + rmAppGroupManager + .terminatingApplicationGroup(groupName, callerUGI.getShortUserName(), + false); + + TerminateApplicationGroupResponse response = recordFactory + .newRecordInstance(TerminateApplicationGroupResponse.class); + return response; + } + + @Override + public ListApplicationGroupsResponse listApplicationGroups( + ListApplicationGroupsRequest request) throws YarnException, IOException { + + ListApplicationGroupsResponse response = recordFactory + .newRecordInstance(ListApplicationGroupsResponse.class); + response.setApplicationGroupList(rmAppGroupManager.listApplicationGroups()); + + return response; + } + private UserGroupInformation getCallerUgi(ApplicationId applicationId, String operation) throws YarnException { UserGroupInformation callerUGI; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 9dc59451499..88e02df507c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.group.RMAppGroup; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetimeMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -67,6 +68,9 @@ private final ConcurrentMap applications = new ConcurrentHashMap(); + private final ConcurrentMap applicationGroups = + new ConcurrentHashMap(); + private final ConcurrentMap nodes = new ConcurrentHashMap(); @@ -483,4 +487,10 @@ public void setContainerQueueLimitCalculator( QueueLimitCalculator limitCalculator) { this.queueLimitCalculator = limitCalculator; } + + @Private + @Unstable + public ConcurrentMap getRMAppGroups() { + return this.applicationGroups; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppGroupManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppGroupManager.java new file mode 100644 index 00000000000..cb5dba0390a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppGroupManager.java @@ -0,0 +1,338 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.records.AppGroupACL; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; +import org.apache.hadoop.yarn.exceptions.ApplicationGroupNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.security.AccessRequest; +import org.apache.hadoop.yarn.security.AdminACLsManager; +import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.group.RMAppGroup; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.group.RMAppGroupImpl; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; + +/** + * This class manages the list of application groups for the resource manager. + */ +public class RMAppGroupManager implements Recoverable { + + private static final Log LOG = LogFactory.getLog(RMAppGroupManager.class); + + private final RMContext rmContext; + private Configuration conf; + private boolean IsRecovered = false; + private final AdminACLsManager adminAclsManager; + private final ConcurrentMap> applicationGroupACLS + = new ConcurrentHashMap>(); + private static String groupTag = "GID-"; + + public RMAppGroupManager(RMContext context, Configuration conf) { + this.rmContext = context; + this.conf = conf; + this.adminAclsManager = new AdminACLsManager(this.conf); + } + + @Override + public void recover(RMState state) throws Exception { + RMStateStore store = rmContext.getStateStore(); + assert store != null; + Map appGroupStates = + state.getApplicationGroupInitState(); + + for (Map.Entry entry : appGroupStates + .entrySet()) { + rmContext.getRMAppGroups() + .putIfAbsent(entry.getKey(), new RMAppGroupImpl(entry.getValue())); + applicationGroupACLS.putIfAbsent(entry.getKey(), + getAppGroupACLFromString(entry.getValue().getPermission())); + } + IsRecovered = true; + } + + public boolean isRecovered(){ + return IsRecovered; + } + + public synchronized void createApplicationGroup(String groupName, + String permission, boolean isPersistent, String createUser, + boolean toSuppressException) throws YarnException { + // Create a future object to capture exceptions from StateStore. + SettableFuture future = SettableFuture.create(); + LOG.info("Start to create application group " + groupName); + + ApplicationGroupStateData appGroupState = ApplicationGroupStateData + .newInstance(groupName, permission, isPersistent, + System.currentTimeMillis(), createUser); + + rmContext.getStateStore() + .createApplicationGroupSynchronously(appGroupState, future); + + try { + Futures.get(future, YarnException.class); + rmContext.getRMAppGroups() + .putIfAbsent(groupName, new RMAppGroupImpl(appGroupState)); + applicationGroupACLS + .putIfAbsent(groupName, getAppGroupACLFromString(permission)); + LOG.info("Create application group " + groupName + " successfully."); + } catch (YarnException ex) { + if (!toSuppressException) { + throw ex; + } + LOG.error( + "Statestore update failed for create application group '" + groupName + + "' with below exception:" + ex.getMessage()); + } + } + + public synchronized void terminatingApplicationGroup(String groupName, + String user, boolean toSuppressException) throws YarnException { + // Create a future object to capture exceptions from StateStore. + SettableFuture future = SettableFuture.create(); + LOG.info("Start to terminating application group " + groupName); + + RMAppGroup group = rmContext.getRMAppGroups().get(groupName); + + if (null == group) { + throw new ApplicationGroupNotFoundException( + "Application group " + groupName + " not exist"); + } + + if (group.getStatus() == YarnApplicationGroupState.TERMINATING + || group.getStatus() == YarnApplicationGroupState.TERMINATED) { + LOG.info( + "Application group " + groupName + " already in terminating status."); + return; + } + + ApplicationGroupStateData appGroupState = ApplicationGroupStateData + .newInstance(group.getGroupName(), group.getPermission(), + group.isPersistent(), YarnApplicationGroupState.TERMINATING, + group.getCreateTime(), group.getTerminateTime(), + group.getCreateUser(), user); + + rmContext.getStateStore() + .updateApplicationGroupSynchronously(appGroupState, future); + + try { + Futures.get(future, YarnException.class); + rmContext.getRMAppGroups().put(groupName, new RMAppGroupImpl(appGroupState)); + LOG.info("Terminating application group " + groupName + " successfully."); + } catch (YarnException ex) { + if (!toSuppressException) { + throw ex; + } + LOG.error( + "Statestore update failed for create application group '" + groupName + + "' with below exception:" + ex.getMessage()); + } + } + + public synchronized void terminateApplicationGroup(String groupName, + String user, boolean toSuppressException) throws YarnException { + // Create a future object to capture exceptions from StateStore. + SettableFuture future = SettableFuture.create(); + + LOG.info("Start to terminate application group " + groupName); + + RMAppGroup group = rmContext.getRMAppGroups().get(groupName); + + if (group.getStatus() == YarnApplicationGroupState.TERMINATED) { + LOG.info( + "Application group " + groupName + " already in terminated status."); + return; + } + + ApplicationGroupStateData appGroupState = ApplicationGroupStateData + .newInstance(group.getGroupName(), group.getPermission(), + group.isPersistent(), YarnApplicationGroupState.TERMINATED, + group.getCreateTime(), System.currentTimeMillis(), + group.getCreateUser(), + user != null ? user : group.getTerminateUser()); + + rmContext.getStateStore() + .updateApplicationGroupSynchronously(appGroupState, future); + + try { + Futures.get(future, YarnException.class); + rmContext.getRMAppGroups().put(groupName, new RMAppGroupImpl(appGroupState)); + LOG.info("Terminate application group " + groupName + " successfully."); + } catch (YarnException ex) { + if (!toSuppressException) { + throw ex; + } + LOG.error( + "Statestore update failed for create application group '" + groupName + + "' with below exception:" + ex.getMessage()); + } + } + + public synchronized boolean checkJoinPermission(UserGroupInformation callerUGI, String groupName){ + Map acls = this.applicationGroupACLS + .get(groupName); + + if(acls == null){ + return false; + } + + if (this.adminAclsManager.isAdmin(callerUGI) + || acls.get(AppGroupACL.JOIN_GROUP).isUserAllowed(callerUGI)) { + return true; + } + + return false; + } + + public synchronized boolean checkAppGroupActive(String groupName) { + + if (null == rmContext.getRMAppGroups().get(groupName)) { + return false; + } + + return YarnApplicationGroupState.ACTIVE == rmContext.getRMAppGroups() + .get(groupName).getStatus(); + } + + public synchronized String getGroupFromTags(Set tags){ + String result = null; + for (String tag : tags) { + if(tag.toUpperCase().startsWith(groupTag)){ + result = tag.substring(groupTag.length()); + break;// pick only the first one match + } + } + return result; + } + + public synchronized boolean checkTerminatePermisson(UserGroupInformation callerUGI, String groupName){ + Map acls = this.applicationGroupACLS + .get(groupName); + + if(acls == null){ + return false; + } + + if (this.adminAclsManager.isAdmin(callerUGI) + || acls.get(AppGroupACL.ADMINISTER_GROUP).isUserAllowed(callerUGI)) { + return true; + } + + return false; + } + + private Map getAppGroupACLFromString( + String permission) { + String[] items = permission.split(";"); + Map result = + new HashMap(); + for (String item : items) { + if (item.startsWith("joinACL:")) { + result.put(AppGroupACL.JOIN_GROUP, + new AccessControlList(item.split(":")[1])); + } else if(item.startsWith("adminACL:")) { + result.put(AppGroupACL.ADMINISTER_GROUP, + new AccessControlList(item.split(":")[1])); + //add default group admin + } else { + LOG.warn("Get unknown item " + item + " just ignore it."); + } + } + return result; + } + + public synchronized void removeApplicationGroup(String groupName, + boolean toSuppressException) throws YarnException { + // Create a future object to capture exceptions from StateStore. + SettableFuture future = SettableFuture.create(); + + LOG.info("Start to remove application group " + groupName); + + RMAppGroup group = rmContext.getRMAppGroups().get(groupName); + + if (null != group) { + ApplicationGroupStateData appGroupState = ApplicationGroupStateData + .newInstance(group.getGroupName(), group.getPermission(), + group.isPersistent(), group.getStatus(), group.getCreateTime(), + System.currentTimeMillis(), group.getCreateUser(), + group.getTerminateUser()); + + rmContext.getStateStore() + .removeApplicationGroupSynchronously(appGroupState, future); + + try { + Futures.get(future, YarnException.class); + rmContext.getRMAppGroups().remove(groupName); + applicationGroupACLS.remove(groupName); + LOG.info("Remove application group " + groupName + " successfully."); + } catch (YarnException ex) { + if (!toSuppressException) { + throw ex; + } + LOG.error("Statestore update failed for create application group '" + + groupName + "' with below exception:" + ex.getMessage()); + } + } else { + LOG.info("Group " + groupName + " already removed."); + } + + } + + public synchronized boolean isApplicationGroupExist(String groupName) { + return rmContext.getRMAppGroups().containsKey(groupName); + } + + public synchronized List listApplicationGroups() + throws YarnException { + List appGroups = new LinkedList(); + for (Map.Entry entry : rmContext.getRMAppGroups() + .entrySet()) { + appGroups.add(ApplicationGroup + .newInstance(entry.getValue().getGroupName(), + entry.getValue().getPermission(), entry.getValue().isPersistent(), + entry.getValue().getStatus())); + } + return appGroups; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index 4f191a6d9e1..692c9e048f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -92,6 +92,10 @@ public static final String DELETE_RESERVATION_REQUEST = "Delete Reservation Request"; public static final String LIST_RESERVATION_REQUEST = "List " + "Reservation Request"; + public static final String CREATE_APPLICATION_GROUP_REQUEST = "Create " + + "Application Group Request"; + public static final String TERMINATING_APPLICATION_GROUP_REQUEST = "Terminate " + + "Application Group Request"; } static String createSuccessLog(String user, String operation, String target, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index b255a304eae..804ae1e33fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.group.RMAppGroup; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetimeMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -65,6 +66,8 @@ RMStateStore getStateStore(); ConcurrentMap getRMApps(); + + ConcurrentMap getRMAppGroups(); ConcurrentMap getSystemCredentialsForApps(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index d7c624d4863..abfa96cf8ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.group.RMAppGroup; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetimeMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -309,6 +310,11 @@ public RMStateStore getStateStore() { return activeServiceContext.getRMApps(); } + @Override + public ConcurrentMap getRMAppGroups() { + return activeServiceContext.getRMAppGroups(); + } + @Override public ConcurrentMap getRMNodes() { return activeServiceContext.getRMNodes(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 60dd94fba1e..18201e36f4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -180,6 +180,8 @@ protected NMLivelinessMonitor nmLivelinessMonitor; protected NodesListManager nodesListManager; protected RMAppManager rmAppManager; + protected RMAppGroupManager rmAppGroupManager; + protected AppGroupMonitor appGroupMonitor; protected ApplicationACLsManager applicationACLsManager; protected QueueACLsManager queueACLsManager; private FederationStateStoreService federationStateStoreService; @@ -484,6 +486,10 @@ protected RMAppManager createRMAppManager() { this.applicationACLsManager, this.conf); } + protected RMAppGroupManager createRMAppGroupManager() { + return new RMAppGroupManager(this.rmContext, this.conf); + } + protected RMApplicationHistoryWriter createRMApplicationHistoryWriter() { return new RMApplicationHistoryWriter(); } @@ -559,6 +565,7 @@ protected static void validateConfigs(Configuration conf) { private ResourceManager rm; private boolean fromActive = false; private StandByTransitionRunnable standByTransitionRunnable; + private boolean enableAppGroup = false; RMActiveServices(ResourceManager rm) { super("RMActiveServices"); @@ -569,6 +576,10 @@ protected static void validateConfigs(Configuration conf) { protected void serviceInit(Configuration configuration) throws Exception { standByTransitionRunnable = new StandByTransitionRunnable(); + enableAppGroup = + conf.getBoolean(YarnConfiguration.RM_APPLICATION_GROUP_ENABLED, + YarnConfiguration.DEFAULT_RM_APPLICATION_GROUP_ENABLED); + conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); rmSecretManagerService = createRMSecretManagerService(); addService(rmSecretManagerService); @@ -707,6 +718,10 @@ protected void serviceInit(Configuration configuration) throws Exception { // Register event handler for RMAppManagerEvents rmDispatcher.register(RMAppManagerEventType.class, rmAppManager); + if(enableAppGroup) { + rmAppGroupManager = createRMAppGroupManager(); + } + clientRM = createClientRMService(); addService(clientRM); rmContext.setClientRMService(clientRM); @@ -737,6 +752,12 @@ protected void serviceInit(Configuration configuration) throws Exception { new RMNMInfo(rmContext, scheduler); + if(enableAppGroup) { + //Start to monitor group + appGroupMonitor = new AppGroupMonitor(rmContext, rmAppGroupManager); + addService(appGroupMonitor); + } + super.serviceInit(conf); } @@ -1272,7 +1293,7 @@ protected ResourceTrackerService createResourceTrackerService() { protected ClientRMService createClientRMService() { return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, - this.rmContext.getRMDelegationTokenSecretManager()); + this.rmContext.getRMDelegationTokenSecretManager(),this.rmAppGroupManager); } protected ApplicationMasterService createApplicationMasterService() { @@ -1393,6 +1414,10 @@ public void recover(RMState state) throws Exception { // recover applications rmAppManager.recover(state); + if(rmAppGroupManager != null) { + rmAppGroupManager.recover(state); + } + setSchedulerRecoveryStartAndWaitTime(state, conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index fbfcdb81410..b8e508d071b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; +import org.apache.commons.lang.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -58,6 +59,7 @@ import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMDelegationTokenIdentifierData; @@ -439,6 +441,24 @@ public synchronized void updateApplicationStateInternal(ApplicationId appId, } } + @Override + protected void storeApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + throw new NotImplementedException(); + } + + @Override + protected void updateApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + throw new NotImplementedException(); + } + + @Override + protected void removeApplicationGroupStateInternal(String groupName) + throws Exception { + throw new NotImplementedException(); + } + @Override public synchronized void storeApplicationAttemptStateInternal( ApplicationAttemptId appAttemptId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index 4f8805c9ed4..589e82024c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -32,6 +32,7 @@ import java.util.Timer; import java.util.TimerTask; +import org.apache.commons.lang.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -57,6 +58,7 @@ import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMDelegationTokenIdentifierData; @@ -599,6 +601,24 @@ protected void updateApplicationStateInternal(ApplicationId appId, storeApplicationStateInternal(appId, appStateData); } + @Override + protected void storeApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + throw new NotImplementedException(); + } + + @Override + protected void updateApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + throw new NotImplementedException(); + } + + @Override + protected void removeApplicationGroupStateInternal(String groupName) + throws Exception { + throw new NotImplementedException(); + } + @Override protected void storeApplicationAttemptStateInternal( ApplicationAttemptId attemptId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index 5041000f0ba..2df01a23c86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import com.google.common.annotations.VisibleForTesting; @@ -114,6 +116,34 @@ public synchronized void updateApplicationStateInternal( state.appState.put(appId, appState); } + @Override + protected void storeApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + state.appGroupInitState.put(groupName, appGroupStateData); + } + + @Override + protected void updateApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + if (state.appGroupInitState.containsKey(groupName)){ + state.appGroupInitState.put(groupName, appGroupStateData); + } else { + throw new YarnRuntimeException("Unexpected Exception. Update group " + + groupName + " not found."); + } + } + + @Override + protected void removeApplicationGroupStateInternal(String groupName) + throws Exception { + if (state.appGroupInitState.containsKey(groupName)){ + state.appGroupInitState.remove(groupName); + } else { + throw new YarnRuntimeException("Unexpected Exception. Update group " + + groupName + " not found."); + } + } + @Override public synchronized void storeApplicationAttemptStateInternal( ApplicationAttemptId appAttemptId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java index 4e134ac9c4c..2bddf5a7722 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; @Unstable @@ -126,6 +127,24 @@ protected void updateApplicationStateInternal(ApplicationId appId, // Do nothing } + @Override + protected void storeApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + // Do nothing + } + + @Override + protected void updateApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + // Do nothing + } + + @Override + protected void removeApplicationGroupStateInternal(String groupName) + throws Exception { + // Do nothing + } + @Override protected void updateApplicationAttemptStateInternal(ApplicationAttemptId attemptId, ApplicationAttemptStateData attemptStateData) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateRemoveAppGroupEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateRemoveAppGroupEvent.java new file mode 100644 index 00000000000..61d9a963e19 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateRemoveAppGroupEvent.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.recovery; + +import com.google.common.util.concurrent.SettableFuture; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; + +public class RMStateRemoveAppGroupEvent extends RMStateStoreEvent { + ApplicationGroupStateData appGroupState; + private SettableFuture future; + + RMStateRemoveAppGroupEvent(ApplicationGroupStateData appGroupState) { + super(RMStateStoreEventType.REMOVE_APP_GROUP); + this.appGroupState = appGroupState; + this.future = null; + } + + RMStateRemoveAppGroupEvent(ApplicationGroupStateData appGroupState, + SettableFuture future) { + super(RMStateStoreEventType.REMOVE_APP_GROUP); + this.appGroupState = appGroupState; + this.future = future; + } + + public ApplicationGroupStateData getAppGroupState() { + return appGroupState; + } + + public SettableFuture getResult() { + return future; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index e8ed0b7ee65..1fecbaa0ef6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -88,6 +89,7 @@ // constants for RM App state and RMDTSecretManagerState. @VisibleForTesting public static final String RM_APP_ROOT = "RMAppRoot"; + public static final String RM_APP_GROUP_ROOT = "RMAppGroupRoot"; protected static final String RM_DT_SECRET_MANAGER_ROOT = "RMDTSecretManagerRoot"; protected static final String RM_DELEGATION_TOKENS_ROOT_ZNODE_NAME = "RMDelegationTokensRoot"; @@ -178,6 +180,18 @@ EnumSet.of(RMStateStoreState.ACTIVE, RMStateStoreState.FENCED), RMStateStoreEventType.REMOVE_RESERVATION, new RemoveReservationAllocationTransition()) + .addTransition(RMStateStoreState.ACTIVE, + EnumSet.of(RMStateStoreState.ACTIVE, RMStateStoreState.FENCED), + RMStateStoreEventType.STORE_APP_GROUP, + new StoreAppGroupTransition()) + .addTransition(RMStateStoreState.ACTIVE, + EnumSet.of(RMStateStoreState.ACTIVE, RMStateStoreState.FENCED), + RMStateStoreEventType.UPDATE_APP_GROUP, + new UpdateAppGroupTransition()) + .addTransition(RMStateStoreState.ACTIVE, + EnumSet.of(RMStateStoreState.ACTIVE, RMStateStoreState.FENCED), + RMStateStoreEventType.REMOVE_APP_GROUP, + new RemoveAppGroupTransition()) .addTransition(RMStateStoreState.ACTIVE, RMStateStoreState.FENCED, RMStateStoreEventType.FENCED) .addTransition(RMStateStoreState.FENCED, RMStateStoreState.FENCED, @@ -577,6 +591,117 @@ public RMStateStoreState transition(RMStateStore store, } } + private static class StoreAppGroupTransition + implements MultipleArcTransition { + @Override + public RMStateStoreState transition(RMStateStore store, + RMStateStoreEvent event) { + if (!(event instanceof RMStateStoreAppGroupEvent)) { + // should never happen + LOG.error("Illegal event type: " + event.getClass()); + return RMStateStoreState.ACTIVE; + } + boolean isFenced = false; + ApplicationGroupStateData appGroupState = + ((RMStateStoreAppGroupEvent) event).getAppGroupState(); + + String groupName = appGroupState.getGroupName(); + + SettableFuture result = + ((RMStateStoreAppGroupEvent) event).getResult(); + + try { + store.storeApplicationGroupStateInternal(groupName, appGroupState); + if (result != null) { + result.set(null); + } + } catch (Exception e) { + String msg = "Error storing app group: " + groupName; + LOG.error(msg, e); + isFenced = store.notifyStoreOperationFailedInternal(e); + if (result != null) { + result.setException(new YarnException(msg, e)); + } + } + return finalState(isFenced); + }; + } + + private static class UpdateAppGroupTransition implements + MultipleArcTransition { + @Override + public RMStateStoreState transition(RMStateStore store, + RMStateStoreEvent event) { + if (!(event instanceof RMStateUpdateAppGroupEvent)) { + // should never happen + LOG.error("Illegal event type: " + event.getClass()); + return RMStateStoreState.ACTIVE; + } + boolean isFenced = false; + ApplicationGroupStateData appGroupState = + ((RMStateUpdateAppGroupEvent) event).getAppGroupState(); + + String groupName = appGroupState.getGroupName(); + SettableFuture result = + ((RMStateUpdateAppGroupEvent) event).getResult(); + + LOG.info("Updating info for app group: " + groupName); + try { + store.updateApplicationGroupStateInternal(groupName, appGroupState); + if (result != null) { + result.set(null); + } + } catch (Exception e) { + String msg = "Error updating app group: " + groupName; + LOG.error(msg, e); + isFenced = store.notifyStoreOperationFailedInternal(e); + if (result != null) { + result.setException(new YarnException(msg, e)); + } + } + return finalState(isFenced); + }; + } + + private static class RemoveAppGroupTransition implements + MultipleArcTransition { + @Override + public RMStateStoreState transition(RMStateStore store, + RMStateStoreEvent event) { + if (!(event instanceof RMStateRemoveAppGroupEvent)) { + // should never happen + LOG.error("Illegal event type: " + event.getClass()); + return RMStateStoreState.ACTIVE; + } + boolean isFenced = false; + ApplicationGroupStateData appGroupState = + ((RMStateRemoveAppGroupEvent) event).getAppGroupState(); + + String groupName = appGroupState.getGroupName(); + SettableFuture result = + ((RMStateRemoveAppGroupEvent) event).getResult(); + + LOG.info("Removing info for app group: " + groupName); + try { + store.removeApplicationGroupStateInternal(groupName); + if (result != null) { + result.set(null); + } + } catch (Exception e) { + String msg = "Error removing app group: " + groupName; + LOG.error(msg, e); + isFenced = store.notifyStoreOperationFailedInternal(e); + if (result != null) { + result.setException(new YarnException(msg, e)); + } + } + return finalState(isFenced); + }; + } + private static RMStateStoreState finalState(boolean isFenced) { return isFenced ? RMStateStoreState.FENCED : RMStateStoreState.ACTIVE; } @@ -645,6 +770,9 @@ public int getDTSequenceNumber() { Map appState = new TreeMap(); + Map appGroupInitState = + new TreeMap(); + RMDTSecretManagerState rmSecretManagerState = new RMDTSecretManagerState(); AMRMTokenSecretManagerState amrmTokenSecretManagerState = null; @@ -656,6 +784,10 @@ public int getDTSequenceNumber() { return appState; } + public Map getApplicationGroupInitState() { + return appGroupInitState; + } + public RMDTSecretManagerState getRMDTSecretManagerState() { return rmSecretManagerState; } @@ -821,6 +953,24 @@ public void updateApplicationStateSynchronously(ApplicationStateData appState, new RMStateUpdateAppEvent(appState, notifyApp, resultFuture)); } + public void createApplicationGroupSynchronously(ApplicationGroupStateData appGroupState, + SettableFuture resultFuture) { + handleStoreEvent( + new RMStateStoreAppGroupEvent(appGroupState, resultFuture)); + } + + public void updateApplicationGroupSynchronously(ApplicationGroupStateData appGroupState, + SettableFuture resultFuture) { + handleStoreEvent( + new RMStateUpdateAppGroupEvent(appGroupState, resultFuture)); + } + + public void removeApplicationGroupSynchronously(ApplicationGroupStateData appGroupState, + SettableFuture resultFuture) { + handleStoreEvent( + new RMStateRemoveAppGroupEvent(appGroupState, resultFuture)); + } + public void updateFencedState() { handleStoreEvent(new RMStateStoreEvent(RMStateStoreEventType.FENCED)); } @@ -835,6 +985,15 @@ protected abstract void storeApplicationStateInternal(ApplicationId appId, protected abstract void updateApplicationStateInternal(ApplicationId appId, ApplicationStateData appStateData) throws Exception; + + protected abstract void storeApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception; + + protected abstract void updateApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception; + + protected abstract void removeApplicationGroupStateInternal(String groupName) + throws Exception; @SuppressWarnings("unchecked") /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreAppGroupEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreAppGroupEvent.java new file mode 100644 index 00000000000..e760e5248f5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreAppGroupEvent.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.recovery; + +import com.google.common.util.concurrent.SettableFuture; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; + +public class RMStateStoreAppGroupEvent extends RMStateStoreEvent { + + private final ApplicationGroupStateData appGroupState; + private SettableFuture future; + + public RMStateStoreAppGroupEvent(ApplicationGroupStateData appGroupState) { + super(RMStateStoreEventType.STORE_APP_GROUP); + this.appGroupState = appGroupState; + this.future = null; + } + + public RMStateStoreAppGroupEvent(ApplicationGroupStateData appGroupState, SettableFuture future) { + super(RMStateStoreEventType.STORE_APP_GROUP); + this.appGroupState = appGroupState; + this.future = future; + } + + public ApplicationGroupStateData getAppGroupState() { + return appGroupState; + } + + public SettableFuture getResult() { + return future; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java index b34634d0659..5d8a09aba0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java @@ -36,4 +36,7 @@ UPDATE_AMRM_TOKEN, STORE_RESERVATION, REMOVE_RESERVATION, + STORE_APP_GROUP, + UPDATE_APP_GROUP, + REMOVE_APP_GROUP, } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppGroupEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppGroupEvent.java new file mode 100644 index 00000000000..56582a4ae8b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppGroupEvent.java @@ -0,0 +1,50 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.recovery; + +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; + +import com.google.common.util.concurrent.SettableFuture; + +public class RMStateUpdateAppGroupEvent extends RMStateStoreEvent { + + private final ApplicationGroupStateData appGroupState; + private SettableFuture future; + + public RMStateUpdateAppGroupEvent(ApplicationGroupStateData appGroupState) { + super(RMStateStoreEventType.UPDATE_APP_GROUP); + this.appGroupState = appGroupState; + this.future = null; + } + + public RMStateUpdateAppGroupEvent(ApplicationGroupStateData appGroupState, + SettableFuture future) { + super(RMStateStoreEventType.UPDATE_APP_GROUP); + this.appGroupState = appGroupState; + this.future = future; + } + + public ApplicationGroupStateData getAppGroupState() { + return appGroupState; + } + + public SettableFuture getResult() { + return future; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 7a791690197..18182f53139 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationGroupStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationAttemptStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationStateDataProto; @@ -47,11 +48,13 @@ import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMDelegationTokenIdentifierData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationGroupStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl; import org.apache.zookeeper.CreateMode; @@ -190,6 +193,7 @@ /* Znode paths */ private String zkRootNodePath; private String rmAppRoot; + private String rmAppGroupRoot; private Map rmAppRootHierarchies; private Map rmDelegationTokenHierarchies; private String rmDTSecretManagerRoot; @@ -296,6 +300,7 @@ public synchronized void initInternal(Configuration conf) YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH); zkRootNodePath = getNodePath(znodeWorkingPath, ROOT_ZNODE_NAME); rmAppRoot = getNodePath(zkRootNodePath, RM_APP_ROOT); + rmAppGroupRoot = getNodePath(zkRootNodePath, RM_APP_GROUP_ROOT); String hierarchiesPath = getNodePath(rmAppRoot, RM_APP_ROOT_HIERARCHIES); rmAppRootHierarchies = new HashMap<>(5); rmAppRootHierarchies.put(0, rmAppRoot); @@ -402,6 +407,7 @@ public synchronized void startInternal() throws Exception { create(dtSequenceNumberPath); create(amrmTokenSecretManagerRoot); create(reservationRoot); + create(rmAppGroupRoot); } private void logRootNodeAcls(String prefix) throws Exception { @@ -517,6 +523,8 @@ public synchronized RMState loadState() throws Exception { loadAMRMTokenSecretManagerState(rmState); // recover reservation state loadReservationSystemState(rmState); + // recover RM applications group + loadRMAppGroupInitState(rmState); return rmState; } @@ -741,6 +749,16 @@ private synchronized void loadRMAppState(RMState rmState) throws Exception { } } + private synchronized void loadRMAppGroupInitState(RMState rmState) throws Exception { + List childNodes = getChildren(rmAppGroupRoot); + for (String childNodeName : childNodes) { + byte[] appGroupData = getData(getNodePath(rmAppGroupRoot, childNodeName)); + ApplicationGroupStateDataPBImpl appGroupState = new ApplicationGroupStateDataPBImpl( + ApplicationGroupStateDataProto.parseFrom(appGroupData)); + rmState.appGroupInitState.put(childNodeName, appGroupState); + } + } + private void loadApplicationAttemptState(ApplicationStateData appState, String appPath) throws Exception { List attempts = getChildren(appPath); @@ -885,6 +903,62 @@ protected synchronized void updateApplicationStateInternal( } } + @Override + protected void storeApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + String nodeCreatePath = getLeafZnodePath(groupName, rmAppGroupRoot, 0, true); + + if (LOG.isDebugEnabled()) { + LOG.debug("Storing info for app group: " + groupName + " at: " + nodeCreatePath); + } + + byte[] stateData = appGroupStateData.getProto().toByteArray(); + if (stateData.length <= zknodeLimit) { + zkManager.safeCreate(nodeCreatePath, stateData, zkAcl, + CreateMode.PERSISTENT, zkAcl, fencingNodePath); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Application state data size for " + groupName + " is " + + stateData.length); + } + throw new StoreLimitException("Application Group" + groupName + + " exceeds the maximum allowed size for application group data. " + + "See yarn.resourcemanager.zk-max-znode-size.bytes."); + } + } + + @Override + protected void updateApplicationGroupStateInternal(String groupName, + ApplicationGroupStateData appGroupStateData) throws Exception { + String nodeUpdatePath = getLeafZnodePath(groupName, rmAppGroupRoot, 0, true); + if (LOG.isDebugEnabled()) { + LOG.debug("Updating info for app group: " + groupName + " at: " + nodeUpdatePath); + } + if (exists(nodeUpdatePath)){ + byte[] stateData = appGroupStateData.getProto().toByteArray(); + zkManager.safeSetData(nodeUpdatePath, stateData, -1, zkAcl, + fencingNodePath); + } else { + throw new YarnRuntimeException("Unexpected Exception. Update group " + + groupName + " not found with path " + nodeUpdatePath); + } + } + + @Override + protected void removeApplicationGroupStateInternal(String groupName) + throws Exception { + String nodeRemovePath = getLeafZnodePath(groupName, rmAppGroupRoot, 0, true); + if (LOG.isDebugEnabled()) { + LOG.debug("Removing info for app group: " + groupName + " at: " + nodeRemovePath); + } + if (exists(nodeRemovePath)){ + zkManager.safeDelete(nodeRemovePath, zkAcl, fencingNodePath); + } else { + throw new YarnRuntimeException("Unexpected Exception. Remove group " + + groupName + " not found with path " + nodeRemovePath); + } + } + /* * Handles store, update and remove application attempt state store * operations. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationGroupStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationGroupStateData.java new file mode 100644 index 00000000000..af237319979 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationGroupStateData.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.recovery.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationGroupStateDataProto; +import org.apache.hadoop.yarn.util.Records; + +/** + * Contains all the state data that needs to be stored persistently + * for an Application group. + */ +@Public +@Unstable +public abstract class ApplicationGroupStateData { + + public static ApplicationGroupStateData newInstance(String groupName, + String permission, boolean persistent, + YarnApplicationGroupState status, long createTime, long terminateTime, + String createUser, String terminateUser) { + ApplicationGroupStateData appGroupState = Records.newRecord(ApplicationGroupStateData.class); + appGroupState.setGroupName(groupName); + appGroupState.setPermission(permission); + appGroupState.setIsPersistent(persistent); + appGroupState.setStatus(status); + appGroupState.setCreateTime(createTime); + appGroupState.setTerminateTime(terminateTime); + appGroupState.setCreateUser(createUser); + appGroupState.setTerminateUser(terminateUser); + return appGroupState; + } + + public static ApplicationGroupStateData newInstance(String groupName, + String permission, boolean persistent, long createTime, String createUser) { + return newInstance(groupName, permission, persistent, + YarnApplicationGroupState.ACTIVE, createTime, 0, createUser, ""); + } + + public abstract ApplicationGroupStateDataProto getProto(); + + public abstract void setGroupName(String groupName); + + public abstract String getGroupName(); + + public abstract void setPermission(String permission); + + public abstract String getPermission(); + + public abstract void setIsPersistent(boolean isPersistent); + + public abstract boolean isPersistent(); + + public abstract void setStatus(YarnApplicationGroupState status); + + public abstract YarnApplicationGroupState getStatus(); + + public abstract void setCreateTime(long time); + + public abstract long getCreateTime(); + + public abstract void setTerminateTime(long time); + + public abstract long getTerminateTime(); + + public abstract void setCreateUser(String user); + + public abstract String getCreateUser(); + + public abstract void setTerminateUser(String user); + + public abstract String getTerminateUser(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationGroupStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationGroupStateDataPBImpl.java new file mode 100644 index 00000000000..28e9c3021d3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationGroupStateDataPBImpl.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb; + + +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationGroupStateProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationGroupStateDataProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.ApplicationGroupStateDataProto; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; + +import com.google.protobuf.TextFormat; + + +public class ApplicationGroupStateDataPBImpl extends ApplicationGroupStateData { + ApplicationGroupStateDataProto proto = + ApplicationGroupStateDataProto.getDefaultInstance(); + ApplicationGroupStateDataProto.Builder builder = null; + boolean viaProto = false; + + public ApplicationGroupStateDataPBImpl() { + builder = ApplicationGroupStateDataProto.newBuilder(); + } + + public ApplicationGroupStateDataPBImpl( + ApplicationGroupStateDataProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public synchronized ApplicationGroupStateDataProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + + @Override + public void setGroupName(String groupName) { + maybeInitBuilder(); + builder.setGroupName(groupName); + } + + @Override + public String getGroupName() { + ApplicationGroupStateDataProtoOrBuilder + p = viaProto ? proto : builder; + return p.getGroupName(); + } + + @Override + public void setPermission(String permission) { + maybeInitBuilder(); + builder.setPermission(permission); + } + + @Override + public String getPermission() { + ApplicationGroupStateDataProtoOrBuilder + p = viaProto ? proto : builder; + return p.getPermission(); + } + + @Override + public void setIsPersistent(boolean isPersistent) { + maybeInitBuilder(); + builder.setPersistent(isPersistent); + } + + @Override + public boolean isPersistent() { + ApplicationGroupStateDataProtoOrBuilder + p = viaProto ? proto : builder; + return p.getPersistent(); + } + + @Override + public void setStatus( + YarnApplicationGroupState status) { + maybeInitBuilder(); + builder.setStatus(convertToProtoFormat(status)); + } + + @Override + public YarnApplicationGroupState getStatus() { + ApplicationGroupStateDataProtoOrBuilder + p = viaProto ? proto : builder; + return convertFromProtoFormat(p.getStatus()); + } + + @Override + public void setCreateTime(long time) { + maybeInitBuilder(); + builder.setCreateTime(time); + } + + @Override + public long getCreateTime() { + ApplicationGroupStateDataProtoOrBuilder + p = viaProto ? proto : builder; + return p.getCreateTime(); + } + + @Override + public void setTerminateTime(long time) { + maybeInitBuilder(); + builder.setTerminateTime(time); + } + + @Override + public long getTerminateTime() { + ApplicationGroupStateDataProtoOrBuilder + p = viaProto ? proto : builder; + return p.getTerminateTime(); + } + + @Override + public void setCreateUser(String user) { + maybeInitBuilder(); + builder.setCreateUser(user); + } + + @Override + public String getCreateUser() { + ApplicationGroupStateDataProtoOrBuilder + p = viaProto ? proto : builder; + return p.getCreateUser(); + } + + @Override + public void setTerminateUser(String user) { + maybeInitBuilder(); + builder.setTerminateUser(user); + } + + @Override + public String getTerminateUser() { + ApplicationGroupStateDataProtoOrBuilder + p = viaProto ? proto : builder; + return p.getTerminateUser(); + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationGroupStateDataProto.newBuilder(proto); + } + viaProto = false; + } + + public static ApplicationGroupStateProto convertToProtoFormat(YarnApplicationGroupState e) { + return ApplicationGroupStateProto.valueOf(e.name()); + } + + public static YarnApplicationGroupState convertFromProtoFormat(ApplicationGroupStateProto e) { + return YarnApplicationGroupState.valueOf(e.name()); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java index bcef85f4d54..c5464b615db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java @@ -29,5 +29,11 @@ FINISHED, FAILED, KILLING, - KILLED + KILLED; + + public static boolean isFinalState(RMAppState state) { + return state == FINISHED + || state == FAILED + || state == KILLED; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/group/RMAppGroup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/group/RMAppGroup.java new file mode 100644 index 00000000000..9e7df7e7642 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/group/RMAppGroup.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.rmapp.group; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.ipc.CallerContext; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.CollectorInfo; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LogAggregationStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.util.Records; + +/** + * The interface to an Application Group in the ResourceManager. Take a + * look at {@link RMAppGroupImpl} for its implementation. This interface + * exposes methods to access various updates in application group status/report. + */ +public interface RMAppGroup { + + void setGroupName(String groupName); + + String getGroupName(); + + void setPermission(String permission); + + String getPermission(); + + void setIsPersistent(boolean isPersistent); + + boolean isPersistent(); + + void setStatus(YarnApplicationGroupState status); + + YarnApplicationGroupState getStatus(); + + void setCreateTime(long status); + + long getCreateTime(); + + void setTerminateTime(long status); + + long getTerminateTime(); + + void setHasRunningApplications(boolean has_running_applications); + + boolean hasRunningApplications(); + + void setCreateUser(String user); + + String getCreateUser(); + + void setTerminateUser(String user); + + String getTerminateUser(); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/group/RMAppGroupImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/group/RMAppGroupImpl.java new file mode 100644 index 00000000000..725688e128e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/group/RMAppGroupImpl.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.rmapp.group; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationGroupStateProto; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; + +@SuppressWarnings({ "rawtypes", "unchecked" }) +public class RMAppGroupImpl implements RMAppGroup { + + private static final Log LOG = LogFactory.getLog(RMAppGroupImpl.class); + + private String groupName; + private String permisson; + private boolean isPersistent; + private YarnApplicationGroupState status; + private long createTime = 0; + private long terminateTime = 0; + private boolean hasRunningApplications = false; + private String createUser; + private String terminateUser; + + public RMAppGroupImpl(String groupName, String permisson, boolean isPersistent, + YarnApplicationGroupState status, long createTime, long terminateTime, + String createUser, String terminateUser){ + setGroupName(groupName); + setPermission(permisson); + setIsPersistent(isPersistent); + setStatus(status); + setCreateTime(createTime); + setTerminateTime(terminateTime); + setCreateUser(createUser); + setTerminateUser(terminateUser); + setHasRunningApplications(false); + } + + public RMAppGroupImpl(ApplicationGroupStateData appGroupStateData){ + setGroupName(appGroupStateData.getGroupName()); + setPermission(appGroupStateData.getPermission()); + setIsPersistent(appGroupStateData.isPersistent()); + setStatus(appGroupStateData.getStatus()); + setCreateTime(appGroupStateData.getCreateTime()); + setTerminateTime(appGroupStateData.getTerminateTime()); + setCreateUser(appGroupStateData.getCreateUser()); + setTerminateUser(appGroupStateData.getTerminateUser()); + setHasRunningApplications(false); + } + + @Override + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + @Override + public String getGroupName() { + return this.groupName; + } + + @Override + public void setPermission(String permission) { + this.permisson = permission; + } + + @Override + public String getPermission() { + return this.permisson; + } + + @Override + public void setIsPersistent(boolean isPersistent) { + this.isPersistent = isPersistent; + } + + @Override + public boolean isPersistent() { + return this.isPersistent; + } + + @Override + public void setStatus(YarnApplicationGroupState status) { + this.status = status; + } + + @Override + public YarnApplicationGroupState getStatus() { + return this.status; + } + + @Override + public void setCreateTime(long createTime) { + this.createTime = createTime; + } + + @Override + public long getCreateTime() { + return this.createTime; + } + + @Override + public void setTerminateTime(long terminateTime) { + this.terminateTime = terminateTime; + } + + @Override + public long getTerminateTime() { + return this.terminateTime; + } + + @Override + public void setHasRunningApplications( + boolean has_running_applications) { + this.hasRunningApplications = has_running_applications; + } + + @Override + public boolean hasRunningApplications() { + return this.hasRunningApplications; + } + + @Override + public void setCreateUser(String user) { + this.createUser = user; + } + + @Override + public String getCreateUser() { + return this.createUser; + } + + @Override + public void setTerminateUser(String user) { + this.terminateUser = user; + } + + @Override + public String getTerminateUser() { + return this.terminateUser; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 5a945daf864..92d4527fbd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -174,6 +174,11 @@ public static final String GET_CONTAINER = "/apps/{appid}/appattempts/{appattemptid}/containers/{containerid}"; + + public static final String APPGROUPS = "/appgroups"; + + public static final String APPGROUPS_GROUPNAME = "/appgroups/{groupname}"; + // ----------------QueryParams for RMWebServiceProtocol---------------- public static final String TIME = "time"; @@ -194,6 +199,7 @@ public static final String MAX_TIME = "maxTime"; public static final String APPATTEMPTID = "appattemptid"; public static final String APPID = "appid"; + public static final String APPGROUPNAME = "groupname"; public static final String LABELS = "labels"; public static final String RESERVATION_ID = "reservation-id"; public static final String START_TIME = "start-time"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 5266581812f..84efc614bc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -61,6 +61,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; @@ -72,6 +73,8 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -86,6 +89,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; @@ -97,10 +102,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationGroup; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -149,6 +156,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppGroupInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppGroupRequest; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppGroupsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; @@ -2476,4 +2486,117 @@ public Void run() throws Exception { .build(); } } + + @POST + @Path(RMWSConsts.APPGROUPS) + @Produces({MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public Response createAppGroup(final AppGroupRequest appGroup, + @Context HttpServletRequest hsr) throws AuthorizationException, + YarnException, InterruptedException, IOException { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + + try { + callerUGI.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, YarnException { + CreateApplicationGroupRequest request = CreateApplicationGroupRequest + .newInstance(appGroup.getGroupName(), appGroup.getPermission(), + appGroup.getPersistent()); + rm.getClientRMService().createApplicationGroup(request); + return null; + } + }); + } catch (Exception e) { + LOG.error("Unable to create app groups from ClientRMService", e); + throw new YarnRuntimeException( + "Unable to create app groups from ClientRMService", e); + } + + return Response.status(Status.OK).build(); + } + + @DELETE + @Path(RMWSConsts.APPGROUPS_GROUPNAME) + @Produces({MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public Response terminateAppGroup( + @PathParam(RMWSConsts.APPGROUPNAME) final String groupName, + @Context HttpServletRequest hsr) throws AuthorizationException, + YarnException, InterruptedException, IOException { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + + try { + callerUGI.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, YarnException { + TerminateApplicationGroupRequest request = + TerminateApplicationGroupRequest.newInstance(groupName); + rm.getClientRMService().terminateApplicationGroup(request); + return null; + } + }); + } catch (Exception e) { + LOG.error("Unable to terminate app groups from ClientRMService", e); + throw new YarnRuntimeException( + "Unable to terminate app groups from ClientRMService", e); + } + + return Response.status(Status.OK).build(); + } + + + @GET + @Path(RMWSConsts.APPGROUPS) + @Produces({MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public AppGroupsInfo getAppGroups( + @Context HttpServletRequest hsr) { + init(); + + AppGroupsInfo allAppGroups = new AppGroupsInfo(); + + ListApplicationGroupsRequest request = + ListApplicationGroupsRequest.newInstance(); + + try { + ListApplicationGroupsResponse response = + rm.getClientRMService().listApplicationGroups(request); + for (ApplicationGroup groupInfo : response.getApplicationGroupList()) { + allAppGroups.add(new AppGroupInfo(groupInfo.getGroupName(), + groupInfo.getPermission(), groupInfo.getIsPersistent(), + groupInfo.getStatus().name())); + } + } catch (Exception e) { + LOG.error("Unable to retrieve app groups from ClientRMService", e); + throw new YarnRuntimeException( + "Unable to retrieve app groups from ClientRMService", e); + } + + return allAppGroups; + } + + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupInfo.java new file mode 100644 index 00000000000..47988d2659c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupInfo.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "appgroup") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppGroupInfo { + + String groupName; + + String permission; + + boolean persistent; + + String status; + + public AppGroupInfo() { + } + + public AppGroupInfo(String groupName, String permission, boolean persistent, String status) { + this.groupName = groupName; + this.permission = permission; + this.persistent = persistent; + this.status = status; + } + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + public String getGroupName() { + return this.groupName; + } + + public void setPermission(String permission) { + this.permission = permission; + } + + public String getPermission() { + return this.permission; + } + + public void setPersistent(boolean persistent) { + this.persistent = persistent; + } + + public boolean getPersistent() { + return this.persistent; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getStatus() { + return this.status; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupRequest.java new file mode 100644 index 00000000000..8fe696c5ca7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupRequest.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "appgroup") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppGroupRequest { + + String groupName; + + String permission; + + boolean persistent; + + public AppGroupRequest() { + } + + public AppGroupRequest(String groupName, String permission, boolean persistent) { + this.groupName = groupName; + this.permission = permission; + this.persistent = persistent; + } + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + public String getGroupName() { + return this.groupName; + } + + public void setPermission(String permission) { + this.permission = permission; + } + + public String getPermission() { + return this.permission; + } + + public void setPersistent(boolean persistent) { + this.persistent = persistent; + } + + public boolean getPersistent() { + return this.persistent; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupsInfo.java new file mode 100644 index 00000000000..c31fe57ea72 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppGroupsInfo.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +@XmlRootElement(name = "appgroups") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppGroupsInfo { + + protected ArrayList appGroup = new ArrayList(); + + public AppGroupsInfo() { + } // JAXB needs this + + public void add(AppGroupInfo appgroupinfo) { + appGroup.add(appgroupinfo); + } + + public ArrayList getApps() { + return appGroup; + } + + public void addAll(ArrayList appGroupsInfo) { + appGroup.addAll(appGroupsInfo); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 247cd2195d9..a1479d77cb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -102,3 +102,14 @@ message RMDelegationTokenIdentifierDataProto { optional YARNDelegationTokenIdentifierProto token_identifier = 1; optional int64 renewDate = 2; } + +message ApplicationGroupStateDataProto { + optional string group_name = 1; + optional string permission = 2; + optional bool persistent = 3; + optional ApplicationGroupStateProto status = 4; + optional int64 create_time = 5; + optional string create_user = 6; + optional int64 terminate_time = 7; + optional string terminate_user = 8; +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 6d2e4f45a15..7b6517d3903 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyListOf; @@ -45,8 +46,10 @@ import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CyclicBarrier; +import com.google.common.collect.Maps; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -54,6 +57,7 @@ import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -78,6 +82,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; @@ -88,6 +94,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -115,6 +122,7 @@ import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.YarnApplicationGroupState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; @@ -128,6 +136,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -136,6 +145,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.group.RMAppGroup; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -280,7 +290,7 @@ public void testNonExistingApplicationReport() throws YarnException { request.setApplicationId(ApplicationId.newInstance(0, 0)); try { rmService.getApplicationReport(request); - Assert.fail(); + fail(); } catch (ApplicationNotFoundException ex) { Assert.assertEquals(ex.getMessage(), "Application with id '" + request.getApplicationId() @@ -377,7 +387,7 @@ public void testGetApplicationAttemptReport() throws YarnException, Assert.assertEquals(attemptId, response.getApplicationAttemptReport() .getApplicationAttemptId()); } catch (ApplicationNotFoundException ex) { - Assert.fail(ex.getMessage()); + fail(ex.getMessage()); } } @@ -421,7 +431,7 @@ public void testGetApplicationAttempts() throws YarnException, IOException { .get(0).getApplicationAttemptId()); } catch (ApplicationNotFoundException ex) { - Assert.fail(ex.getMessage()); + fail(ex.getMessage()); } } @@ -442,7 +452,7 @@ public void testGetContainerReport() throws YarnException, IOException { Assert.assertEquals(containerId, response.getContainerReport() .getContainerId()); } catch (ApplicationNotFoundException ex) { - Assert.fail(ex.getMessage()); + fail(ex.getMessage()); } } @@ -461,7 +471,7 @@ public void testGetContainers() throws YarnException, IOException { Assert.assertEquals(containerId, response.getContainerList().get(0) .getContainerId()); } catch (ApplicationNotFoundException ex) { - Assert.fail(ex.getMessage()); + fail(ex.getMessage()); } } @@ -504,7 +514,7 @@ public void testForceKillNonExistingApplication() throws YarnException { KillApplicationRequest.newInstance(applicationId); try { rmService.forceKillApplication(request); - Assert.fail(); + fail(); } catch (ApplicationNotFoundException ex) { Assert.assertEquals(ex.getMessage(), "Trying to kill an absent " + @@ -690,7 +700,7 @@ public void handle(Event event) {} try { rmService.submitApplication(submitRequest1); } catch (YarnException e) { - Assert.fail("Exception is not expected."); + fail("Exception is not expected."); } RMApp app1 = rmContext.getRMApps().get(appId1); Assert.assertNotNull("app doesn't exist", app1); @@ -710,7 +720,7 @@ public void handle(Event event) {} try { rmService.submitApplication(submitRequest2); } catch (YarnException e) { - Assert.fail("Exception is not expected."); + fail("Exception is not expected."); } RMApp app2 = rmContext.getRMApps().get(appId2); Assert.assertNotNull("app doesn't exist", app2); @@ -721,7 +731,7 @@ public void handle(Event event) {} try { rmService.submitApplication(submitRequest2); } catch (YarnException e) { - Assert.fail("Exception is not expected."); + fail("Exception is not expected."); } GetApplicationsRequest getAllAppsRequest = @@ -1198,7 +1208,7 @@ private ResourceManager setupResourceManager() { // allow plan follower to synchronize Thread.sleep(1050); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } return rm; } @@ -1214,7 +1224,7 @@ private ReservationSubmissionRequest submitReservationTestHelper( reservationID = clientService.getNewReservation(newReservationRequest) .getReservationId(); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } ReservationSubmissionRequest sRequest = ReservationSystemTestUtil.createSimpleReservationRequest(reservationID, @@ -1222,7 +1232,7 @@ private ReservationSubmissionRequest submitReservationTestHelper( try { sResponse = clientService.submitReservation(sRequest); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } Assert.assertNotNull(sResponse); Assert.assertNotNull(reservationID); @@ -1246,7 +1256,7 @@ public void testCreateReservation() { try { clientService.submitReservation(sRequest); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } // Submit the reservation with the same reservation id but different @@ -1257,7 +1267,7 @@ public void testCreateReservation() { sRequest.setReservationDefinition(rDef); try { clientService.submitReservation(sRequest); - Assert.fail("Reservation submission should fail if a duplicate " + fail("Reservation submission should fail if a duplicate " + "reservation id is used, but the reservation definition has been " + "updated."); } catch (Exception e) { @@ -1295,7 +1305,7 @@ public void testUpdateReservation() { try { uResponse = clientService.updateReservation(uRequest); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } Assert.assertNotNull(uResponse); System.out.println("Update reservation response: " + uResponse); @@ -1322,7 +1332,7 @@ public void testListReservationsByReservationId() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); @@ -1357,7 +1367,7 @@ public void testListReservationsByTimeInterval() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); @@ -1371,7 +1381,7 @@ public void testListReservationsByTimeInterval() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); @@ -1412,7 +1422,7 @@ public void testListReservationsByInvalidTimeInterval() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); @@ -1427,7 +1437,7 @@ public void testListReservationsByInvalidTimeInterval() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } Assert.assertNotNull(response); Assert.assertEquals(1, response.getReservationAllocationState().size()); @@ -1456,7 +1466,7 @@ public void testListReservationsByTimeIntervalContainingNoReservations() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } // Ensure all reservations are filtered out. @@ -1476,7 +1486,7 @@ public void testListReservationsByTimeIntervalContainingNoReservations() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } // Ensure all reservations are filtered out. @@ -1494,7 +1504,7 @@ public void testListReservationsByTimeIntervalContainingNoReservations() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } // Ensure all reservations are filtered out. @@ -1509,7 +1519,7 @@ public void testListReservationsByTimeIntervalContainingNoReservations() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } // Ensure all reservations are filtered out. @@ -1538,7 +1548,7 @@ public void testReservationDelete() { try { dResponse = clientService.deleteReservation(dRequest); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } Assert.assertNotNull(dResponse); System.out.println("Delete reservation response: " + dResponse); @@ -1552,7 +1562,7 @@ public void testReservationDelete() { try { response = clientService.listReservations(request); } catch (Exception e) { - Assert.fail(e.getMessage()); + fail(e.getMessage()); } Assert.assertNotNull(response); Assert.assertEquals(0, response.getReservationAllocationState().size()); @@ -1751,7 +1761,7 @@ public void testUpdateApplicationPriorityRequest() throws Exception { Priority.newInstance(appPriority)); try { rmService.updateApplicationPriority(updateRequest); - Assert.fail("ApplicationNotFoundException should be thrown " + fail("ApplicationNotFoundException should be thrown " + "for invalid application id"); } catch (ApplicationNotFoundException e) { // Expected @@ -1894,4 +1904,101 @@ public void handle(Event event) { rmService.getApplications(request).getApplicationList().size()); rmService.setDisplayPerUserApps(false); } + + @Test (timeout = 30000) + public void testAppGroup() throws Exception { + YarnScheduler yarnScheduler = mockYarnScheduler(); + RMContext rmContext = mock(RMContext.class); + mockRMContext(yarnScheduler, rmContext); + RMStateStore stateStore = new MemoryRMStateStore(); + final ConcurrentMap applicationGroups = + new ConcurrentHashMap(); + when(rmContext.getStateStore()).thenReturn(stateStore); + when(rmContext.getRMAppGroups()).thenReturn(applicationGroups); + RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, + null, mock(ApplicationACLsManager.class), new Configuration()); + when(rmContext.getDispatcher().getEventHandler()).thenReturn( + new EventHandler() { + public void handle(Event event) {} + }); + doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext) + .getRMTimelineCollectorManager(); + + ApplicationId appId1 = getApplicationId(100); + + ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); + when( + mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); + + QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class); + when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), + any(QueueACL.class), any(RMApp.class), any(String.class), + Matchers.>any())) + .thenReturn(true); + + Configuration conf = new Configuration(); + RMAppGroupManager rmAppGroupManager = + new RMAppGroupManager(rmContext, conf); + + ClientRMService rmService = + new ClientRMService(rmContext, yarnScheduler, appManager, + mockAclsManager, mockQueueACLsManager, null, rmAppGroupManager); + rmService.init(conf); + + ListApplicationGroupsRequest listRequest = ListApplicationGroupsRequest + .newInstance(); + + ListApplicationGroupsResponse list; + + try { + list = rmService.listApplicationGroups(listRequest); + assertTrue(list.getApplicationGroupList().isEmpty()); + } catch (YarnException ex) { + fail( + "get application groups failed with exception " + ex.getMessage()); + } + + CreateApplicationGroupRequest request = CreateApplicationGroupRequest + .newInstance("TestGroup", "joinACL:hadoop;adminACL:hadoop", true); + + try { + rmService.createApplicationGroup(request); + } catch (YarnException ex) { + fail( + "Create application group failed with exception " + ex.getMessage()); + } + + try { + list = rmService.listApplicationGroups(listRequest); + assertEquals(list.getApplicationGroupList().get(0).getGroupName(), "TestGroup"); + assertEquals(list.getApplicationGroupList().get(0).getStatus(), + YarnApplicationGroupState.ACTIVE); + assertEquals(list.getApplicationGroupList().get(0).getPermission(), + "joinACL:hadoop;adminACL:hadoop"); + assertTrue(list.getApplicationGroupList().get(0).getIsPersistent()); + } catch (YarnException ex) { + fail( + "get application groups failed with exception " + ex.getMessage()); + } + + try { + TerminateApplicationGroupRequest terminateRequest = TerminateApplicationGroupRequest + .newInstance("TestGroup"); + rmService.terminateApplicationGroup(terminateRequest); + } catch (YarnException ex) { + fail( + "get application groups failed with exception " + ex.getMessage()); + } + + try { + list = rmService.listApplicationGroups(listRequest); + assertEquals(list.getApplicationGroupList().get(0).getGroupName(), "TestGroup"); + assertEquals(list.getApplicationGroupList().get(0).getStatus(), + YarnApplicationGroupState.TERMINATING); + } catch (YarnException ex) { + fail( + "get application groups failed with exception " + ex.getMessage()); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index fbacac0cc86..e844725c3c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.RetryNTimes; @@ -45,6 +47,7 @@ import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Event; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.records.Version; @@ -54,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationGroupStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -63,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.group.RMAppGroupImpl; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.security.MasterKeyData; @@ -1489,4 +1494,82 @@ public void testDelegationTokenNodeWithSplitChangeAcrossRestarts() tokensWithIndex, sequenceNumber, 3); store.close(); } + + @Test + public void testAppGroupStatus() throws Exception { + TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester(); + long submitTime = System.currentTimeMillis(); + long startTime = submitTime + 1234; + Configuration conf = new YarnConfiguration(); + + // Create store with app node split set as 1. + RMStateStore store = zkTester.getRMStateStore(createConfForAppNodeSplit(0)); + TestDispatcher dispatcher = new TestDispatcher(); + store.setRMDispatcher(dispatcher); + RMContext rmContext = mock(RMContext.class); + when(rmContext.getStateStore()).thenReturn(store); + + SettableFuture future = SettableFuture.create(); + ApplicationGroupStateData appGroupState = ApplicationGroupStateData + .newInstance("TestGroup", "joinACL:mtpkrbrs;adminACL:mtpkrbrs", true, + System.currentTimeMillis(), "mtpkrbrs"); + + rmContext.getStateStore() + .updateApplicationGroupSynchronously(appGroupState, future); + + try { + Futures.get(future, YarnException.class); + fail("Should fail for this call as no Test group exist."); + } catch (YarnException ex) { + //ignore + } + + future = SettableFuture.create(); + rmContext.getStateStore() + .removeApplicationGroupSynchronously(appGroupState, future); + + try { + Futures.get(future, YarnException.class); + fail("Should fail for this call as no TestGroup exist."); + } catch (YarnException ex) { + //ignore + } + + + + future = SettableFuture.create(); + + rmContext.getStateStore() + .createApplicationGroupSynchronously(appGroupState, future); + + try { + Futures.get(future, YarnException.class); + } catch (YarnException ex) { + fail( + "Create application group failed with exception " + ex.getMessage()); + } + + rmContext.getStateStore() + .updateApplicationGroupSynchronously(appGroupState, future); + + try { + Futures.get(future, YarnException.class); + } catch (YarnException ex) { + fail( + "Create update group failed with exception " + ex.getMessage()); + } + + future = SettableFuture.create(); + rmContext.getStateStore() + .removeApplicationGroupSynchronously(appGroupState, future); + + try { + Futures.get(future, YarnException.class); + } catch (YarnException ex) { + fail( + "Create remove group failed with exception " + ex.getMessage()); + } + + store.close(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java index 71de6b470e9..904f34f89c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java @@ -25,6 +25,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; @@ -61,6 +63,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; @@ -77,6 +81,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -303,6 +309,25 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( return clientRMProxy.updateApplicationTimeouts(request); } + @Override + public CreateApplicationGroupResponse createApplicationGroup( + CreateApplicationGroupRequest request) throws YarnException, IOException { + return clientRMProxy.createApplicationGroup(request); + } + + @Override + public TerminateApplicationGroupResponse terminateApplicationGroup( + TerminateApplicationGroupRequest request) + throws YarnException, IOException { + return clientRMProxy.terminateApplicationGroup(request); + } + + @Override + public ListApplicationGroupsResponse listApplicationGroups( + ListApplicationGroupsRequest request) throws YarnException, IOException { + return clientRMProxy.listApplicationGroups(request); + } + @VisibleForTesting public void setRMClient(ApplicationClientProtocol clientRM) { this.clientRMProxy = clientRM; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 3a36eec66ac..b9129e1bb94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; @@ -66,6 +68,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; @@ -82,6 +86,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -709,4 +715,23 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( throw new NotImplementedException(); } + @Override + public CreateApplicationGroupResponse createApplicationGroup( + CreateApplicationGroupRequest request) throws YarnException, IOException { + throw new NotImplementedException(); + } + + @Override + public TerminateApplicationGroupResponse terminateApplicationGroup( + TerminateApplicationGroupRequest request) + throws YarnException, IOException { + throw new NotImplementedException(); + } + + @Override + public ListApplicationGroupsResponse listApplicationGroups( + ListApplicationGroupsRequest request) throws YarnException, IOException { + throw new NotImplementedException(); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index fd2c610c7fe..faccfe3e995 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -36,6 +36,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; @@ -72,6 +74,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; @@ -88,6 +92,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -403,6 +409,28 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( return pipeline.getRootInterceptor().updateApplicationTimeouts(request); } + @Override + public CreateApplicationGroupResponse createApplicationGroup( + CreateApplicationGroupRequest request) throws YarnException, IOException { + RequestInterceptorChainWrapper pipeline = getInterceptorChain(); + return pipeline.getRootInterceptor().createApplicationGroup(request); + } + + @Override + public TerminateApplicationGroupResponse terminateApplicationGroup( + TerminateApplicationGroupRequest request) + throws YarnException, IOException { + RequestInterceptorChainWrapper pipeline = getInterceptorChain(); + return pipeline.getRootInterceptor().terminateApplicationGroup(request); + } + + @Override + public ListApplicationGroupsResponse listApplicationGroups( + ListApplicationGroupsRequest request) throws YarnException, IOException { + RequestInterceptorChainWrapper pipeline = getInterceptorChain(); + return pipeline.getRootInterceptor().listApplicationGroups(request); + } + private RequestInterceptorChainWrapper getInterceptorChain() throws IOException { String user = UserGroupInformation.getCurrentUser().getUserName(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java index c403bd5006c..f4e9e5c90ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java @@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CreateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; @@ -58,6 +60,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ListApplicationGroupsResponse; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; @@ -74,6 +78,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupRequest; +import org.apache.hadoop.yarn.api.protocolrecords.TerminateApplicationGroupResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; @@ -264,4 +270,23 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( throws YarnException, IOException { return getNextInterceptor().updateApplicationTimeouts(request); } + + @Override + public CreateApplicationGroupResponse createApplicationGroup( + CreateApplicationGroupRequest request) throws YarnException, IOException { + return getNextInterceptor().createApplicationGroup(request); + } + + @Override + public TerminateApplicationGroupResponse terminateApplicationGroup( + TerminateApplicationGroupRequest request) + throws YarnException, IOException { + return getNextInterceptor().terminateApplicationGroup(request); + } + + @Override + public ListApplicationGroupsResponse listApplicationGroups( + ListApplicationGroupsRequest request) throws YarnException, IOException { + return getNextInterceptor().listApplicationGroups(request); + } } -- 2.23.0.windows.1