commit 0b0e0a2edd65ed336c7770dcc89a674396a2e8c7 Author: Vinod Kumar Vavilapalli Date: Tue Jul 16 12:13:03 2013 -0700 YARN-930. Bootstrap ApplicationHistoryService module diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index 5e6b9e2..157ca5c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -60,6 +60,7 @@ containermanagement_protocol.proto server/yarn_server_resourcemanager_service_protos.proto server/resourcemanager_administration_protocol.proto + src/main/proto/application_history_service.proto ${project.build.directory}/generated-sources/java diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationHistoryProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationHistoryProtocol.java new file mode 100644 index 0000000..491aad3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationHistoryProtocol.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + *

The protocol between clients and the ApplicationHistoryService + * to get information on completed applications etc.

+ */ +public interface ApplicationHistoryProtocol { + + /** + *

The interface used by clients to get a report of an Application from + * the HistoryServer.

+ * + *

The client, via {@link GetApplicationReportRequest} provides the + * {@link ApplicationId} of the application.

+ * + *

In secure mode,the HistoryServer verifies access to the + * application etc. before accepting the request.

+ * + *

The HistoryServer responds with a + * {@link GetApplicationReportResponse} which includes the + * {@link ApplicationReport} for the application.

+ * + *

If the user does not have VIEW_APP access then the + * following fields in the report will be set to stubbed values: + *

+ * + * @param request request for an application report + * @return application report + * @throws YarnException + */ + public GetApplicationReportResponse getApplicationReport( + GetApplicationReportRequest request) throws YarnException; + + /** + *

The interface used by clients to get a report of all Applications + * in the cluster from the HistoryServer.

+ * + *

The HistoryServer responds with a + * {@link GetApplicationsRequest} which includes the + * {@link ApplicationReport} for all the applications.

+ * + *

If the user does not have VIEW_APP access for an + * application then the corresponding report will be filtered as + * described in {@link #getApplicationReport(GetApplicationReportRequest)}. + *

+ * + * @param request request for report on all running applications + * @return report on all running applications + * @throws YarnException + */ + public GetApplicationsResponse getApplications( + GetApplicationsRequest request) throws YarnException; + + /** + *

The interface used by clients to get delegation token, enabling the + * containers to be able to talk to the service using those tokens. + * + *

The HistoryServer responds with the delegation token + * {@link Token} that can be used by the client to speak to this + * service. + * @param request request to get a delegation token for the client. + * @return delegation token that can be used to talk to this service + * @throws YarnException + */ + public GetDelegationTokenResponse getDelegationToken( + GetDelegationTokenRequest request) throws YarnException; + + /** + * Renew an existing delegation token. + * + * @param request the delegation token to be renewed. + * @return the new expiry time for the delegation token. + * @throws YarnException + */ + @Private + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnException; + + /** + * Cancel an existing delegation token. + * + * @param request the delegation token to be cancelled. + * @return an empty response. + * @throws YarnException + */ + @Private + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnException; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/application_history_service.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/application_history_service.proto new file mode 100644 index 0000000..4524e09 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/application_history_service.proto @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "ApplicationHistoryProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; + +import "Security.proto"; +import "yarn_service_protos.proto"; + +service ApplicationHistoryProtocolService { + rpc getApplicationReport (GetApplicationReportRequestProto) returns (GetApplicationReportResponseProto); + rpc getAllApplications (GetAllApplicationsRequestProto) returns (GetAllApplicationsResponseProto); + rpc getDelegationToken(hadoop.common.GetDelegationTokenRequestProto) returns (hadoop.common.GetDelegationTokenResponseProto); + rpc renewDelegationToken(hadoop.common.RenewDelegationTokenRequestProto) returns (hadoop.common.RenewDelegationTokenResponseProto); + rpc cancelDelegationToken(hadoop.common.CancelDelegationTokenRequestProto) returns (hadoop.common.CancelDelegationTokenResponseProto); +} + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml new file mode 100644 index 0000000..77cc881 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -0,0 +1,43 @@ + + + + 4.0.0 + + hadoop-yarn-server + org.apache.hadoop + 3.0.0-SNAPSHOT + .. + + hadoop-yarn-server-applicationhistoryserver + 3.0.0-SNAPSHOT + + + + ${project.parent.basedir} + + + + + org.apache.hadoop + hadoop-yarn-server-common + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AHSClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AHSClientService.java new file mode 100644 index 0000000..8bed68b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AHSClientService.java @@ -0,0 +1,76 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; + +public class AHSClientService extends AbstractService implements + ApplicationHistoryProtocol { + + public AHSClientService() { + super(AHSClientService.class.getName()); + } + + @Override + public GetApplicationReportResponse getApplicationReport( + GetApplicationReportRequest request) throws YarnException { + // TODO Auto-generated method stub + return null; + } + + @Override + public GetApplicationsResponse + getApplications(GetApplicationsRequest request) throws YarnException { + // TODO Auto-generated method stub + return null; + } + + @Override + public GetDelegationTokenResponse getDelegationToken( + GetDelegationTokenRequest request) throws YarnException { + // TODO Auto-generated method stub + return null; + } + + @Override + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnException { + // TODO Auto-generated method stub + return null; + } + + @Override + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnException { + // TODO Auto-generated method stub + return null; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AHSWebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AHSWebServer.java new file mode 100644 index 0000000..fba5ef8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AHSWebServer.java @@ -0,0 +1,29 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import org.apache.hadoop.service.AbstractService; + +public class AHSWebServer extends AbstractService { + + public AHSWebServer() { + super(AHSWebServer.class.getName()); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java new file mode 100644 index 0000000..0c20d49 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -0,0 +1,42 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.CompositeService; + +/** + * History server that keeps track of all types of history in the cluster. + * Application specific history to start with. + */ +public class ApplicationHistoryServer extends CompositeService { + + public ApplicationHistoryServer() { + super(ApplicationHistoryServer.class.getName()); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + super.serviceInit(conf); + AHSClientService ahsClientService = new AHSClientService(); + addService(ahsClientService); + AHSWebServer webServer = new AHSWebServer(); + addService(webServer); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml index 6a2a73e..2bd682b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml @@ -47,5 +47,6 @@ hadoop-yarn-server-web-proxy hadoop-yarn-server-resourcemanager hadoop-yarn-server-tests + hadoop-yarn-server-applicationhistoryservice