diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index affbe03..600bb54 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -96,6 +96,8 @@ server/resourcemanager_administration_protocol.proto application_history_client.proto server/application_history_server.proto + client_SCM_protocol.proto + SCM_Admin_protocol.proto ${project.build.directory}/generated-sources/java diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java new file mode 100644 index 0000000..74efbe9 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + *

+ * The protocol between clients and the SharedCacheManager to claim + * and release resources in the shared cache. + *

+ */ +@Public +@Stable +public interface ClientSCMProtocol { + /** + *

+ * The interface used by clients to claim a resource with the + * SharedCacheManager. The client uses a checksum to identify the + * resource and an {@link ApplicationId} to identify which application will be + * using the resource. + *

+ * + *

+ * The SharedCacheManager responds with whether or not the + * resource exists in the cache. If the resource exists, a Path + * to the resource in the shared cache is returned. If the resource does not + * exist, the response is empty. + *

+ * + * @param request request to claim a resource in the shared cache + * @return response indicating if the resource is already in the cache + * @throws YarnException + * @throws IOException + */ + public UseSharedCacheResourceResponse use( + UseSharedCacheResourceRequest request) throws YarnException, IOException; + + /** + *

+ * The interface used by clients to release a resource with the + * SharedCacheManager. This method is called once an application + * is no longer using a claimed resource in the shared cache. The client uses + * a checksum to identify the resource and an {@link ApplicationId} to + * identify which application is releasing the resource. + *

+ * + *

+ * Note: This method is an optimization and the client is not required to call + * it for correctness. + *

+ * + *

+ * Currently the SharedCacheManager sends an empty response. + *

+ * + * @param request request to release a resource in the shared cache + * @return (empty) response on releasing the resource + * @throws YarnException + * @throws IOException + */ + public ReleaseSharedCacheResourceResponse release( + ReleaseSharedCacheResourceRequest request) throws YarnException, IOException; + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocolPB.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocolPB.java new file mode 100644 index 0000000..b0a9fb5 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocolPB.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api; + +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.proto.ClientSCMProtocol.ClientSCMProtocolService; + +@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.api.ClientSCMProtocolPB", + protocolVersion = 1) +public interface ClientSCMProtocolPB extends + ClientSCMProtocolService.BlockingInterface { + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocol.java new file mode 100644 index 0000000..1bc618d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocol.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + *

+ * The protocol between administrators and the SharedCacheManager + *

+ */ +@Public +@Stable +public interface SCMAdminProtocol { + /** + *

+ * The method used by administrators to ask SCM to run cleaner task right away + *

+ * + * @param request request SharedCacheManager to run a cleaner task + * @return SharedCacheManager returns an empty response + * on success and throws an exception on rejecting the request + * @throws YarnException + * @throws IOException + */ + public RunSharedCacheCleanerTaskResponse runCleanerTask( + RunSharedCacheCleanerTaskRequest request) throws YarnException, IOException; + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocolPB.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocolPB.java new file mode 100644 index 0000000..1883c16c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/SCMAdminProtocolPB.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api; + +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.proto.SCMAdminProtocol.SCMAdminProtocolService; + +@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.api.SCMAdminProtocolPB", + protocolVersion = 1) +public interface SCMAdminProtocolPB extends + SCMAdminProtocolService.BlockingInterface { +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceRequest.java new file mode 100644 index 0000000..993451b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceRequest.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + *

The request from clients to release a resource in the shared cache.

+ */ +@Public +@Stable +public abstract class ReleaseSharedCacheResourceRequest { + + /** + * Get the ApplicationId of the resource to be released. + * + * @return ApplicationId + */ + @Public + @Stable + public abstract ApplicationId getAppId(); + + /** + * Set the ApplicationId of the resource to be released. + * + * @param id ApplicationId + */ + @Public + @Stable + public abstract void setAppId(ApplicationId id); + + /** + * Get the key of the resource to be released. + * + * @return key + */ + @Public + @Stable + public abstract String getResourceKey(); + + /** + * Set the key of the resource to be released. + * + * @param key unique identifier for the resource + */ + @Public + @Stable + public abstract void setResourceKey(String key); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceResponse.java new file mode 100644 index 0000000..c36f53d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ReleaseSharedCacheResourceResponse.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The response to clients from the SharedCacheManager when + * releasing a resource in the shared cache. + *

+ * + *

+ * Currently, this is empty. + *

+ */ +@Public +@Stable +public abstract class ReleaseSharedCacheResourceResponse { +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskRequest.java new file mode 100644 index 0000000..7407b28 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskRequest.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The request from admin to ask the SharedCacheManager to run + * cleaner service right away. + *

+ * + *

+ * Currently, this is empty. + *

+ */ +@Public +@Stable +public abstract class RunSharedCacheCleanerTaskRequest { +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskResponse.java new file mode 100644 index 0000000..e68baa7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RunSharedCacheCleanerTaskResponse.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The response to admin from the SharedCacheManager when + * is asked to run the cleaner service. + *

+ * + *

+ * Currently, this is empty. + *

+ */ +@Public +@Stable +public abstract class RunSharedCacheCleanerTaskResponse { + + /** + * Get whether or not the shared cache manager has accepted the request. + * Shared cache manager will reject the request if there is an ongoing task + * + * @return boolean True if the resource has been accepted, false otherwise. + */ + @Public + @Stable + public abstract boolean getAccepted(); + + /** + * Set whether or not the shared cache manager has accepted the request + * Shared cache manager will reject the request if there is an ongoing task + * + * @param b True if the resource has been accepted, false otherwise. + */ + @Public + @Stable + public abstract void setAccepted(boolean b); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceRequest.java new file mode 100644 index 0000000..a0869fe --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceRequest.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + *

+ * The request from clients to the SharedCacheManager that claims a + * resource in the shared cache. + *

+ */ +@Public +@Stable +public abstract class UseSharedCacheResourceRequest { + + /** + * Get the ApplicationId of the resource to be used. + * + * @return ApplicationId + */ + @Public + @Stable + public abstract ApplicationId getAppId(); + + /** + * Set the ApplicationId of the resource to be used. + * + * @param id ApplicationId + */ + @Public + @Stable + public abstract void setAppId(ApplicationId id); + + /** + * Get the key of the resource to be used. + * + * @return key + */ + @Public + @Stable + public abstract String getResourceKey(); + + /** + * Set the key of the resource to be used. + * + * @param key unique identifier for the resource + */ + @Public + @Stable + public abstract void setResourceKey(String key); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceResponse.java new file mode 100644 index 0000000..35d1a70 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UseSharedCacheResourceResponse.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The response from the SharedCacheManager to the client that indicates whether + * a requested resource exists in the cache. + *

+ */ +@Public +@Stable +public abstract class UseSharedCacheResourceResponse { + + /** + * Get the Path corresponding to the requested resource in the + * shared cache. + * + * @return String A Path if the resource exists in the shared + * cache, null otherwise + */ + @Public + @Stable + public abstract String getPath(); + + /** + * Set the Path corresponding to a resource in the shared cache. + * + * @param p A Path corresponding to a resource in the shared + * cache + */ + @Public + @Stable + public abstract void setPath(String p); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/SCM_Admin_protocol.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/SCM_Admin_protocol.proto new file mode 100644 index 0000000..4e46c57 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/SCM_Admin_protocol.proto @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "SCMAdminProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.yarn; + +import "yarn_service_protos.proto"; + +service SCMAdminProtocolService { + rpc runCleanerTask (RunSharedCacheCleanerTaskRequestProto) returns (RunSharedCacheCleanerTaskResponseProto); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_SCM_protocol.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_SCM_protocol.proto new file mode 100644 index 0000000..fbc3c42 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_SCM_protocol.proto @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "ClientSCMProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.yarn; + +import "yarn_service_protos.proto"; + +service ClientSCMProtocolService { + rpc use (UseSharedCacheResourceRequestProto) returns (UseSharedCacheResourceResponseProto); + rpc release (ReleaseSharedCacheResourceRequestProto) returns (ReleaseSharedCacheResourceResponseProto); +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index a1f6d2e..490f810 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -285,3 +285,31 @@ message GetContainersRequestProto { message GetContainersResponseProto { repeated ContainerReportProto containers = 1; } + +////////////////////////////////////////////////////// +/////// client_SCM_Protocol ////////////////////////// +////////////////////////////////////////////////////// + +message UseSharedCacheResourceRequestProto { + optional ApplicationIdProto applicationId = 1; + optional string resourceKey = 2; +} + +message UseSharedCacheResourceResponseProto { + optional string path = 1; +} + +message ReleaseSharedCacheResourceRequestProto { + optional ApplicationIdProto applicationId = 1; + optional string resourceKey = 2; +} + +message ReleaseSharedCacheResourceResponseProto { +} + +message RunSharedCacheCleanerTaskRequestProto { +} + +message RunSharedCacheCleanerTaskResponseProto { + optional bool accepted = 1; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientSCMProtocolPBClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientSCMProtocolPBClientImpl.java new file mode 100644 index 0000000..58af3c4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientSCMProtocolPBClientImpl.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.impl.pb.client; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.api.ClientSCMProtocol; +import org.apache.hadoop.yarn.api.ClientSCMProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReleaseSharedCacheResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReleaseSharedCacheResourceResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceRequestProto; + +import com.google.protobuf.ServiceException; + +public class ClientSCMProtocolPBClientImpl implements ClientSCMProtocol, + Closeable { + + private ClientSCMProtocolPB proxy; + + public ClientSCMProtocolPBClientImpl(long clientVersion, + InetSocketAddress addr, Configuration conf) throws IOException { + RPC.setProtocolEngine(conf, ClientSCMProtocolPB.class, + ProtobufRpcEngine.class); + proxy = RPC.getProxy(ClientSCMProtocolPB.class, clientVersion, addr, conf); + } + + @Override + public void close() { + if (this.proxy != null) { + RPC.stopProxy(this.proxy); + } + } + + @Override + public UseSharedCacheResourceResponse use( + UseSharedCacheResourceRequest request) throws YarnException, IOException { + UseSharedCacheResourceRequestProto requestProto = + ((UseSharedCacheResourceRequestPBImpl) request).getProto(); + try { + return new UseSharedCacheResourceResponsePBImpl(proxy.use(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public ReleaseSharedCacheResourceResponse release( + ReleaseSharedCacheResourceRequest request) throws YarnException, + IOException { + ReleaseSharedCacheResourceRequestProto requestProto = + ((ReleaseSharedCacheResourceRequestPBImpl) request).getProto(); + try { + return new ReleaseSharedCacheResourceResponsePBImpl(proxy.release(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/SCMAdminProtocolPBClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/SCMAdminProtocolPBClientImpl.java new file mode 100644 index 0000000..aabc2a3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/SCMAdminProtocolPBClientImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.impl.pb.client; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.api.ClientSCMProtocolPB; +import org.apache.hadoop.yarn.api.SCMAdminProtocol; +import org.apache.hadoop.yarn.api.SCMAdminProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; + +import com.google.protobuf.ServiceException; + +public class SCMAdminProtocolPBClientImpl implements SCMAdminProtocol, + Closeable { + + private SCMAdminProtocolPB proxy; + + public SCMAdminProtocolPBClientImpl(long clientVersion, + InetSocketAddress addr, Configuration conf) throws IOException { + RPC.setProtocolEngine(conf, SCMAdminProtocolPB.class, + ProtobufRpcEngine.class); + proxy = RPC.getProxy(SCMAdminProtocolPB.class, clientVersion, addr, conf); + } + + @Override + public void close() { + if (this.proxy != null) { + RPC.stopProxy(this.proxy); + } + } + + @Override + public RunSharedCacheCleanerTaskResponse runCleanerTask( + RunSharedCacheCleanerTaskRequest request) throws YarnException, + IOException { + YarnServiceProtos.RunSharedCacheCleanerTaskRequestProto requestProto = + ((RunSharedCacheCleanerTaskRequestPBImpl) request).getProto(); + try { + return new RunSharedCacheCleanerTaskResponsePBImpl(proxy.runCleanerTask(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientSCMProtocolPBServiceImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientSCMProtocolPBServiceImpl.java new file mode 100644 index 0000000..8c4e549 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientSCMProtocolPBServiceImpl.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.impl.pb.service; + +import java.io.IOException; + +import org.apache.hadoop.yarn.api.ClientSCMProtocol; +import org.apache.hadoop.yarn.api.ClientSCMProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReleaseSharedCacheResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReleaseSharedCacheResourceResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceResponseProto; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + +public class ClientSCMProtocolPBServiceImpl implements ClientSCMProtocolPB { + + private ClientSCMProtocol real; + + public ClientSCMProtocolPBServiceImpl(ClientSCMProtocol impl) { + this.real = impl; + } + + @Override + public UseSharedCacheResourceResponseProto use(RpcController controller, + UseSharedCacheResourceRequestProto proto) throws ServiceException { + UseSharedCacheResourceRequestPBImpl request = + new UseSharedCacheResourceRequestPBImpl(proto); + try { + UseSharedCacheResourceResponse response = real.use(request); + return ((UseSharedCacheResourceResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public ReleaseSharedCacheResourceResponseProto release( + RpcController controller, ReleaseSharedCacheResourceRequestProto proto) + throws ServiceException { + ReleaseSharedCacheResourceRequestPBImpl request = + new ReleaseSharedCacheResourceRequestPBImpl(proto); + try { + ReleaseSharedCacheResourceResponse response = real.release(request); + return ((ReleaseSharedCacheResourceResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/SCMAdminProtocolPBServiceImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/SCMAdminProtocolPBServiceImpl.java new file mode 100644 index 0000000..937e363 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/SCMAdminProtocolPBServiceImpl.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.impl.pb.service; + +import java.io.IOException; + +import org.apache.hadoop.yarn.api.SCMAdminProtocol; +import org.apache.hadoop.yarn.api.SCMAdminProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskResponseProto; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + +public class SCMAdminProtocolPBServiceImpl implements SCMAdminProtocolPB { + + private SCMAdminProtocol real; + + public SCMAdminProtocolPBServiceImpl(SCMAdminProtocol impl) { + this.real = impl; + } + + @Override + public RunSharedCacheCleanerTaskResponseProto runCleanerTask(RpcController controller, + RunSharedCacheCleanerTaskRequestProto proto) throws ServiceException { + RunSharedCacheCleanerTaskRequestPBImpl request = + new RunSharedCacheCleanerTaskRequestPBImpl(proto); + try { + RunSharedCacheCleanerTaskResponse response = real.runCleanerTask(request); + return ((RunSharedCacheCleanerTaskResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceRequestPBImpl.java new file mode 100644 index 0000000..6db61a1 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceRequestPBImpl.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceRequestProtoOrBuilder; + +public class ReleaseSharedCacheResourceRequestPBImpl extends + ReleaseSharedCacheResourceRequest { + ReleaseSharedCacheResourceRequestProto proto = + ReleaseSharedCacheResourceRequestProto.getDefaultInstance(); + ReleaseSharedCacheResourceRequestProto.Builder builder = null; + boolean viaProto = false; + + private ApplicationId applicationId = null; + + public ReleaseSharedCacheResourceRequestPBImpl() { + builder = ReleaseSharedCacheResourceRequestProto.newBuilder(); + } + + public ReleaseSharedCacheResourceRequestPBImpl( + ReleaseSharedCacheResourceRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public ReleaseSharedCacheResourceRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public ApplicationId getAppId() { + ReleaseSharedCacheResourceRequestProtoOrBuilder p = + viaProto ? proto : builder; + if (this.applicationId != null) { + return this.applicationId; + } + if (!p.hasApplicationId()) { + return null; + } + this.applicationId = convertFromProtoFormat(p.getApplicationId()); + return this.applicationId; + } + + @Override + public void setAppId(ApplicationId id) { + maybeInitBuilder(); + if (id == null) + builder.clearApplicationId(); + this.applicationId = id; + } + + @Override + public String getResourceKey() { + ReleaseSharedCacheResourceRequestProtoOrBuilder p = + viaProto ? proto : builder; + return (p.hasResourceKey()) ? p.getResourceKey() : null; + } + + @Override + public void setResourceKey(String key) { + maybeInitBuilder(); + if (key == null) { + builder.clearResourceKey(); + return; + } + builder.setResourceKey(key); + } + + private void mergeLocalToBuilder() { + if (applicationId != null) { + builder.setApplicationId(convertToProtoFormat(this.applicationId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ReleaseSharedCacheResourceRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { + return new ApplicationIdPBImpl(p); + } + + private ApplicationIdProto convertToProtoFormat(ApplicationId t) { + return ((ApplicationIdPBImpl) t).getProto(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceResponsePBImpl.java new file mode 100644 index 0000000..559f2c8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ReleaseSharedCacheResourceResponsePBImpl.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReleaseSharedCacheResourceResponseProto; + +public class ReleaseSharedCacheResourceResponsePBImpl extends + ReleaseSharedCacheResourceResponse { + ReleaseSharedCacheResourceResponseProto proto = + ReleaseSharedCacheResourceResponseProto.getDefaultInstance(); + ReleaseSharedCacheResourceResponseProto.Builder builder = null; + boolean viaProto = false; + + public ReleaseSharedCacheResourceResponsePBImpl() { + builder = ReleaseSharedCacheResourceResponseProto.newBuilder(); + } + + public ReleaseSharedCacheResourceResponsePBImpl( + ReleaseSharedCacheResourceResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public ReleaseSharedCacheResourceResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ReleaseSharedCacheResourceResponseProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskRequestPBImpl.java new file mode 100644 index 0000000..d432332 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskRequestPBImpl.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskRequestProto; + +public class RunSharedCacheCleanerTaskRequestPBImpl extends + RunSharedCacheCleanerTaskRequest { + RunSharedCacheCleanerTaskRequestProto proto = + RunSharedCacheCleanerTaskRequestProto.getDefaultInstance(); + RunSharedCacheCleanerTaskRequestProto.Builder builder = null; + boolean viaProto = false; + + public RunSharedCacheCleanerTaskRequestPBImpl() { + builder = RunSharedCacheCleanerTaskRequestProto.newBuilder(); + } + + public RunSharedCacheCleanerTaskRequestPBImpl( + RunSharedCacheCleanerTaskRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public RunSharedCacheCleanerTaskRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RunSharedCacheCleanerTaskRequestProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskResponsePBImpl.java new file mode 100644 index 0000000..c7da352 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RunSharedCacheCleanerTaskResponsePBImpl.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RunSharedCacheCleanerTaskResponseProtoOrBuilder; + +public class RunSharedCacheCleanerTaskResponsePBImpl extends + RunSharedCacheCleanerTaskResponse { + RunSharedCacheCleanerTaskResponseProto proto = + RunSharedCacheCleanerTaskResponseProto.getDefaultInstance(); + RunSharedCacheCleanerTaskResponseProto.Builder builder = null; + boolean viaProto = false; + + public RunSharedCacheCleanerTaskResponsePBImpl() { + builder = RunSharedCacheCleanerTaskResponseProto.newBuilder(); + } + + public RunSharedCacheCleanerTaskResponsePBImpl( + RunSharedCacheCleanerTaskResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public boolean getAccepted() { + RunSharedCacheCleanerTaskResponseProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasAccepted()) ? p.getAccepted() : false; + } + + @Override + public void setAccepted(boolean b) { + maybeInitBuilder(); + builder.setAccepted(b); + } + + public RunSharedCacheCleanerTaskResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RunSharedCacheCleanerTaskResponseProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceRequestPBImpl.java new file mode 100644 index 0000000..8499b9f --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceRequestPBImpl.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceRequestProtoOrBuilder; + +public class UseSharedCacheResourceRequestPBImpl extends + UseSharedCacheResourceRequest { + UseSharedCacheResourceRequestProto proto = UseSharedCacheResourceRequestProto + .getDefaultInstance(); + UseSharedCacheResourceRequestProto.Builder builder = null; + boolean viaProto = false; + + private ApplicationId applicationId = null; + + public UseSharedCacheResourceRequestPBImpl() { + builder = UseSharedCacheResourceRequestProto.newBuilder(); + } + + public UseSharedCacheResourceRequestPBImpl( + UseSharedCacheResourceRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public UseSharedCacheResourceRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public ApplicationId getAppId() { + UseSharedCacheResourceRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.applicationId != null) { + return this.applicationId; + } + if (!p.hasApplicationId()) { + return null; + } + this.applicationId = convertFromProtoFormat(p.getApplicationId()); + return this.applicationId; + } + + @Override + public void setAppId(ApplicationId id) { + maybeInitBuilder(); + if (id == null) + builder.clearApplicationId(); + this.applicationId = id; + } + + @Override + public String getResourceKey() { + UseSharedCacheResourceRequestProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasResourceKey()) ? p.getResourceKey() : null; + } + + @Override + public void setResourceKey(String key) { + maybeInitBuilder(); + if (key == null) { + builder.clearResourceKey(); + return; + } + builder.setResourceKey(key); + } + + private void mergeLocalToBuilder() { + if (applicationId != null) { + builder.setApplicationId(convertToProtoFormat(this.applicationId)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = UseSharedCacheResourceRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { + return new ApplicationIdPBImpl(p); + } + + private ApplicationIdProto convertToProtoFormat(ApplicationId t) { + return ((ApplicationIdPBImpl) t).getProto(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceResponsePBImpl.java new file mode 100644 index 0000000..5a9c14b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UseSharedCacheResourceResponsePBImpl.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.UseSharedCacheResourceResponseProtoOrBuilder; + +public class UseSharedCacheResourceResponsePBImpl extends + UseSharedCacheResourceResponse { + UseSharedCacheResourceResponseProto proto = + UseSharedCacheResourceResponseProto + .getDefaultInstance(); + UseSharedCacheResourceResponseProto.Builder builder = null; + boolean viaProto = false; + + public UseSharedCacheResourceResponsePBImpl() { + builder = UseSharedCacheResourceResponseProto.newBuilder(); + } + + public UseSharedCacheResourceResponsePBImpl( + UseSharedCacheResourceResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public UseSharedCacheResourceResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public String getPath() { + UseSharedCacheResourceResponseProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasPath()) ? p.getPath() : null; + } + + @Override + public void setPath(String path) { + maybeInitBuilder(); + if (path == null) { + builder.clearPath(); + return; + } + builder.setPath(path); + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = UseSharedCacheResourceResponseProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index acf330f..ca01d31 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -135,6 +135,7 @@ yarn_server_common_service_protos.proto yarn_server_common_service_protos.proto ResourceTracker.proto + NMCacheUploader_SCM_protocol.proto ${project.build.directory}/generated-sources/java diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocol.java new file mode 100644 index 0000000..89d78c0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocol.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; + +/** + *

+ * The protocol between a NodeManager's + * SharedCacheUploadService and the + * SharedCacheManager. + *

+ */ +@Public +@Stable +public interface NMCacheUploaderSCMProtocol { + /** + *

+ * The method used by the NodeManager's SharedCacheUploadService + * to notify the shared cache manager of a newly cached resource. + *

+ * + *

+ * The SharedCacheManager responds with whether or not the + * NodeManager should delete the uploaded file. + *

+ * + * @param request notify the shared cache manager of a newly uploaded resource + * to the shared cache + * @return response indicating if the newly uploaded resource should be + * deleted + * @throws YarnException + * @throws IOException + */ + public NotifySCMResponse notify(NotifySCMRequest request) + throws YarnException, IOException; + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocolPB.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocolPB.java new file mode 100644 index 0000000..13e055b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/NMCacheUploaderSCMProtocolPB.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api; + +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.proto.NMCacheUploaderSCMProtocol.NMCacheUploaderSCMProtocolService; + +@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocolPB", + protocolVersion = 1) +public interface NMCacheUploaderSCMProtocolPB extends + NMCacheUploaderSCMProtocolService.BlockingInterface { + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/NMCacheUploaderSCMProtocolPBClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/NMCacheUploaderSCMProtocolPBClientImpl.java new file mode 100644 index 0000000..1cebf6d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/NMCacheUploaderSCMProtocolPBClientImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.impl.pb.client; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMRequestProto; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocol; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocolPB; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NotifySCMRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NotifySCMResponsePBImpl; + +import com.google.protobuf.ServiceException; + +public class NMCacheUploaderSCMProtocolPBClientImpl implements + NMCacheUploaderSCMProtocol, Closeable { + + private NMCacheUploaderSCMProtocolPB proxy; + + public NMCacheUploaderSCMProtocolPBClientImpl(long clientVersion, + InetSocketAddress addr, Configuration conf) throws IOException { + RPC.setProtocolEngine(conf, NMCacheUploaderSCMProtocolPB.class, + ProtobufRpcEngine.class); + proxy = + RPC.getProxy(NMCacheUploaderSCMProtocolPB.class, clientVersion, addr, + conf); + } + + @Override + public void close() { + if (this.proxy != null) { + RPC.stopProxy(this.proxy); + } + } + + @Override + public NotifySCMResponse notify(NotifySCMRequest request) + throws YarnException, IOException { + NotifySCMRequestProto requestProto = + ((NotifySCMRequestPBImpl) request).getProto(); + try { + return new NotifySCMResponsePBImpl(proxy.notify(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/NMCacheUploaderSCMProtocolPBServiceImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/NMCacheUploaderSCMProtocolPBServiceImpl.java new file mode 100644 index 0000000..7bac95c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/NMCacheUploaderSCMProtocolPBServiceImpl.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.impl.pb.service; + +import java.io.IOException; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMResponseProto; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocol; +import org.apache.hadoop.yarn.server.api.NMCacheUploaderSCMProtocolPB; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NotifySCMRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NotifySCMResponsePBImpl; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + +public class NMCacheUploaderSCMProtocolPBServiceImpl implements + NMCacheUploaderSCMProtocolPB { + + private NMCacheUploaderSCMProtocol real; + + public NMCacheUploaderSCMProtocolPBServiceImpl(NMCacheUploaderSCMProtocol impl) { + this.real = impl; + } + + @Override + public NotifySCMResponseProto notify(RpcController controller, + NotifySCMRequestProto proto) throws ServiceException { + NotifySCMRequestPBImpl request = new NotifySCMRequestPBImpl(proto); + try { + NotifySCMResponse response = real.notify(request); + return ((NotifySCMResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMRequest.java new file mode 100644 index 0000000..c1a508b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMRequest.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The request from clients to the SharedCacheManager that claims a + * resource in the shared cache. + *

+ */ +@Public +@Stable +public abstract class NotifySCMRequest { + + /** + * Get the filename of the resource that was just uploaded to the shared + * cache. + * + * @return the filename + */ + @Public + @Stable + public abstract String getFileName(); + + /** + * Set the filename of the resource that was just uploaded to the shared + * cache. + * + * @param id filename + */ + @Public + @Stable + public abstract void setFilename(String filename); + + /** + * Get the key of the resource that was just uploaded to the + * shared cache. + * + * @return key + */ + @Public + @Stable + public abstract String getResourceKey(); + + /** + * Set the key of the resource that was just uploaded to the + * shared cache. + * + * @param key unique identifier for the resource + */ + @Public + @Stable + public abstract void setResourceKey(String key); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMResponse.java new file mode 100644 index 0000000..c17b430 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NotifySCMResponse.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

+ * The response from the SharedCacheManager to the NodeManager that indicates + * whether the NodeManager needs to delete the cached resource it was sending + * the notification for. + *

+ */ +@Public +@Stable +public abstract class NotifySCMResponse { + + /** + * Get whether or not the shared cache manager has accepted the notified + * resource (i.e. the uploaded file should remain in the cache). + * + * @return boolean True if the resource has been accepted, false otherwise. + */ + @Public + @Stable + public abstract boolean getAccepted(); + + /** + * Set whether or not the shared cache manager has accepted the notified + * resource (i.e. the uploaded file should remain in the cache). + * + * @param b True if the resource has been accepted, false otherwise. + */ + @Public + @Stable + public abstract void setAccepted(boolean b); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMRequestPBImpl.java new file mode 100644 index 0000000..6bddb96 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMRequestPBImpl.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMRequest; + +public class NotifySCMRequestPBImpl extends + NotifySCMRequest { + NotifySCMRequestProto proto = NotifySCMRequestProto + .getDefaultInstance(); + NotifySCMRequestProto.Builder builder = null; + boolean viaProto = false; + + public NotifySCMRequestPBImpl() { + builder = NotifySCMRequestProto.newBuilder(); + } + + public NotifySCMRequestPBImpl(NotifySCMRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public NotifySCMRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public String getResourceKey() { + NotifySCMRequestProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasResourceKey()) ? p.getResourceKey() : null; + } + + @Override + public void setResourceKey(String key) { + maybeInitBuilder(); + if (key == null) { + builder.clearResourceKey(); + return; + } + builder.setResourceKey(key); + } + + @Override + public String getFileName() { + NotifySCMRequestProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasFilename()) ? p.getFilename() : null; + } + + @Override + public void setFilename(String filename) { + maybeInitBuilder(); + if (filename == null) { + builder.clearFilename(); + return; + } + builder.setFilename(filename); + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = NotifySCMRequestProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMResponsePBImpl.java new file mode 100644 index 0000000..cb5f978 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NotifySCMResponsePBImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NotifySCMResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.NotifySCMResponse; + +public class NotifySCMResponsePBImpl extends + NotifySCMResponse { + NotifySCMResponseProto proto = NotifySCMResponseProto + .getDefaultInstance(); + NotifySCMResponseProto.Builder builder = null; + boolean viaProto = false; + + public NotifySCMResponsePBImpl() { + builder = NotifySCMResponseProto.newBuilder(); + } + + public NotifySCMResponsePBImpl( +NotifySCMResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public NotifySCMResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public boolean getAccepted() { + NotifySCMResponseProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasAccepted()) ? p.getAccepted() : null; + } + + @Override + public void setAccepted(boolean b) { + maybeInitBuilder(); + builder.setAccepted(b); + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = NotifySCMResponseProto.newBuilder(proto); + } + viaProto = false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/NMCacheUploader_SCM_protocol.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/NMCacheUploader_SCM_protocol.proto new file mode 100644 index 0000000..9e4c33d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/NMCacheUploader_SCM_protocol.proto @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "NMCacheUploaderSCMProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.yarn; + +import "yarn_server_common_service_protos.proto"; + +service NMCacheUploaderSCMProtocolService { + rpc notify(NotifySCMRequestProto) returns (NotifySCMResponseProto); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto index aab4383..d4585c9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto @@ -68,3 +68,12 @@ message NMContainerStatusProto { optional string diagnostics = 5 [default = "N/A"]; optional int32 container_exit_status = 6; } + +message NotifySCMRequestProto { + optional string resource_key = 1; + optional string filename = 2; +} + +message NotifySCMResponseProto { + optional bool accepted = 1; +}