commit cd54bcc85ff8f1d84597a32dc66729199d8190f1 Author: Eric Yang Date: Fri Nov 17 16:17:36 2017 -0500 YARN-7530. Moved hadoop-yarn-services-api project to a module of hadoop-yarn-services. (Contributed by Eric Yang) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml deleted file mode 100644 index b89146a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml deleted file mode 100644 index ddea2a1..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ /dev/null @@ -1,130 +0,0 @@ - - - 4.0.0 - - org.apache.hadoop - hadoop-yarn-applications - 3.1.0-SNAPSHOT - - hadoop-yarn-services-api - Apache Hadoop YARN Services API - jar - Hadoop YARN REST APIs for services - - - - - - - src/main/resources - true - - - src/main/scripts/ - true - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - - - development - ${project.url} - - - - - - - - - - test-jar - - - - - - - - - - - - - - org.apache.hadoop - hadoop-yarn-services-core - - - org.apache.hadoop - hadoop-yarn-api - - - org.apache.hadoop - hadoop-yarn-common - - - org.apache.hadoop - hadoop-common - - - org.slf4j - slf4j-api - - - org.eclipse.jetty - jetty-webapp - - - com.google.inject - guice - - - javax.ws.rs - jsr311-api - - - org.mockito - mockito-all - test - - - - - - - - org.apache.hadoop - hadoop-common - test-jar - - - junit - junit - test - - - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java deleted file mode 100644 index 1bb6c93..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.webapp; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.VersionInfo; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.api.records.Service; -import org.apache.hadoop.yarn.service.api.records.ServiceState; -import org.apache.hadoop.yarn.service.api.records.ServiceStatus; -import org.apache.hadoop.yarn.service.client.ServiceClient; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED; -import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; - -/** - * The rest API endpoints for users to manage services on YARN. - */ -@Singleton -@Path(CONTEXT_ROOT) -public class ApiServer { - - public ApiServer() { - super(); - } - - @Inject - public ApiServer(Configuration conf) { - super(); - } - - private static final Logger LOG = - LoggerFactory.getLogger(ApiServer.class); - private static Configuration YARN_CONFIG = new YarnConfiguration(); - private static ServiceClient SERVICE_CLIENT; - - static { - init(); - } - - // initialize all the common resources - order is important - private static void init() { - SERVICE_CLIENT = new ServiceClient(); - SERVICE_CLIENT.init(YARN_CONFIG); - SERVICE_CLIENT.start(); - } - - @GET - @Path(VERSION) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response getVersion() { - String version = VersionInfo.getBuildVersion(); - LOG.info(version); - return Response.ok("{ \"hadoop_version\": \"" + version + "\"}").build(); - } - - @POST - @Path(SERVICE_ROOT_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response createService(Service service) { - LOG.info("POST: createService = {}", service); - ServiceStatus serviceStatus = new ServiceStatus(); - try { - ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service); - LOG.info("Successfully created service " + service.getName() - + " applicationId = " + applicationId); - serviceStatus.setState(ACCEPTED); - serviceStatus.setUri( - CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service - .getName()); - return Response.status(Status.ACCEPTED).entity(serviceStatus).build(); - } catch (IllegalArgumentException e) { - serviceStatus.setDiagnostics(e.getMessage()); - return Response.status(Status.BAD_REQUEST).entity(serviceStatus) - .build(); - } catch (Exception e) { - String message = "Failed to create service " + service.getName(); - LOG.error(message, e); - serviceStatus.setDiagnostics(message + ": " + e.getMessage()); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(serviceStatus).build(); - } - } - - @GET - @Path(SERVICE_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response getService(@PathParam(SERVICE_NAME) String appName) { - LOG.info("GET: getService for appName = {}", appName); - ServiceStatus serviceStatus = new ServiceStatus(); - try { - Service app = SERVICE_CLIENT.getStatus(appName); - return Response.ok(app).build(); - } catch (IllegalArgumentException e) { - serviceStatus.setDiagnostics(e.getMessage()); - serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID); - return Response.status(Status.NOT_FOUND).entity(serviceStatus) - .build(); - } catch (Exception e) { - LOG.error("Get service failed", e); - serviceStatus - .setDiagnostics("Failed to retrieve service: " + e.getMessage()); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(serviceStatus).build(); - } - } - - @DELETE - @Path(SERVICE_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response deleteService(@PathParam(SERVICE_NAME) String appName) { - LOG.info("DELETE: deleteService for appName = {}", appName); - return stopService(appName, true); - } - - private Response stopService(String appName, boolean destroy) { - try { - SERVICE_CLIENT.actionStop(appName, destroy); - if (destroy) { - SERVICE_CLIENT.actionDestroy(appName); - LOG.info("Successfully deleted service {}", appName); - } else { - LOG.info("Successfully stopped service {}", appName); - } - return Response.status(Status.OK).build(); - } catch (ApplicationNotFoundException e) { - ServiceStatus serviceStatus = new ServiceStatus(); - serviceStatus.setDiagnostics( - "Service " + appName + " is not found in YARN: " + e.getMessage()); - return Response.status(Status.BAD_REQUEST).entity(serviceStatus) - .build(); - } catch (Exception e) { - ServiceStatus serviceStatus = new ServiceStatus(); - serviceStatus.setDiagnostics(e.getMessage()); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(serviceStatus).build(); - } - } - - @PUT - @Path(COMPONENT_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN }) - public Response updateComponent(@PathParam(SERVICE_NAME) String appName, - @PathParam(COMPONENT_NAME) String componentName, Component component) { - - if (component.getNumberOfContainers() < 0) { - return Response.status(Status.BAD_REQUEST).entity( - "Service = " + appName + ", Component = " + component.getName() - + ": Invalid number of containers specified " + component - .getNumberOfContainers()).build(); - } - ServiceStatus status = new ServiceStatus(); - try { - Map original = SERVICE_CLIENT.flexByRestService(appName, - Collections.singletonMap(component.getName(), - component.getNumberOfContainers())); - status.setDiagnostics( - "Updating component (" + componentName + ") size from " + original - .get(componentName) + " to " + component.getNumberOfContainers()); - return Response.ok().entity(status).build(); - } catch (YarnException | IOException e) { - status.setDiagnostics(e.getMessage()); - return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status) - .build(); - } - } - - @PUT - @Path(SERVICE_PATH) - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response updateService(@PathParam(SERVICE_NAME) String appName, - Service updateServiceData) { - LOG.info("PUT: updateService for app = {} with data = {}", appName, - updateServiceData); - - // Ignore the app name provided in updateServiceData and always use appName - // path param - updateServiceData.setName(appName); - - // For STOP the app should be running. If already stopped then this - // operation will be a no-op. For START it should be in stopped state. - // If already running then this operation will be a no-op. - if (updateServiceData.getState() != null - && updateServiceData.getState() == ServiceState.STOPPED) { - return stopService(appName, false); - } - - // If a START is requested - if (updateServiceData.getState() != null - && updateServiceData.getState() == ServiceState.STARTED) { - return startService(appName); - } - - // If new lifetime value specified then update it - if (updateServiceData.getLifetime() != null - && updateServiceData.getLifetime() > 0) { - return updateLifetime(appName, updateServiceData); - } - - // If nothing happens consider it a no-op - return Response.status(Status.NO_CONTENT).build(); - } - - private Response updateLifetime(String appName, Service updateAppData) { - ServiceStatus status = new ServiceStatus(); - try { - String newLifeTime = - SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); - status.setDiagnostics( - "Service (" + appName + ")'s lifeTime is updated to " + newLifeTime - + ", " + updateAppData.getLifetime() - + " seconds remaining"); - return Response.ok(status).build(); - } catch (Exception e) { - String message = - "Failed to update service (" + appName + ")'s lifetime to " - + updateAppData.getLifetime(); - LOG.error(message, e); - status.setDiagnostics(message + ": " + e.getMessage()); - return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status) - .build(); - } - } - - private Response startService(String appName) { - ServiceStatus status = new ServiceStatus(); - try { - SERVICE_CLIENT.actionStart(appName); - LOG.info("Successfully started service " + appName); - status.setDiagnostics("Service " + appName + " is successfully started."); - status.setState(ServiceState.ACCEPTED); - return Response.ok(status).build(); - } catch (Exception e) { - String message = "Failed to start service " + appName; - status.setDiagnostics(message + ": " + e.getMessage()); - LOG.info(message, e); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(status).build(); - } - } - - /** - * Used by negative test case. - * - * @param mockServerClient - A mocked version of ServiceClient - */ - public static void setServiceClient(ServiceClient mockServerClient) { - SERVICE_CLIENT = mockServerClient; - SERVICE_CLIENT.init(YARN_CONFIG); - SERVICE_CLIENT.start(); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java deleted file mode 100644 index f4acd94..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.webapp; - -import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.AuthenticationFilterInitializer; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; -import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.eclipse.jetty.webapp.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY; -import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*; - -/** - * This class launches the web service using Hadoop HttpServer2 (which uses - * an embedded Jetty container). This is the entry point to your service. - * The Java command used to launch this app should call the main method. - */ -public class ApiServerWebApp extends AbstractService { - private static final Logger logger = LoggerFactory - .getLogger(ApiServerWebApp.class); - private static final String SEP = ";"; - - // REST API server for YARN native services - private HttpServer2 apiServer; - private InetSocketAddress bindAddress; - - public static void main(String[] args) throws IOException { - ApiServerWebApp apiWebApp = new ApiServerWebApp(); - try { - apiWebApp.init(new YarnConfiguration()); - apiWebApp.serviceStart(); - } catch (Exception e) { - logger.error("Got exception starting", e); - apiWebApp.close(); - } - } - - public ApiServerWebApp() { - super(ApiServerWebApp.class.getName()); - } - - @Override - protected void serviceStart() throws Exception { - bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS, - DEFAULT_API_SERVER_ADDRESS, DEFAULT_API_SERVER_PORT); - logger.info("YARN API server running on " + bindAddress); - if (UserGroupInformation.isSecurityEnabled()) { - doSecureLogin(getConfig()); - } - startWebApp(); - super.serviceStart(); - } - - @Override - protected void serviceStop() throws Exception { - if (apiServer != null) { - apiServer.stop(); - } - super.serviceStop(); - } - - private void doSecureLogin(org.apache.hadoop.conf.Configuration conf) - throws IOException { - SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB, - YarnConfiguration.RM_PRINCIPAL, bindAddress.getHostName()); - addFilters(conf); - } - - private void addFilters(org.apache.hadoop.conf.Configuration conf) { - // Always load pseudo authentication filter to parse "user.name" in an URL - // to identify a HTTP request's user. - boolean hasHadoopAuthFilterInitializer = false; - String filterInitializerConfKey = "hadoop.http.filter.initializers"; - Class[] initializersClasses = - conf.getClasses(filterInitializerConfKey); - List targets = new ArrayList(); - if (initializersClasses != null) { - for (Class initializer : initializersClasses) { - if (initializer.getName().equals( - AuthenticationFilterInitializer.class.getName())) { - hasHadoopAuthFilterInitializer = true; - break; - } - targets.add(initializer.getName()); - } - } - if (!hasHadoopAuthFilterInitializer) { - targets.add(AuthenticationFilterInitializer.class.getName()); - conf.set(filterInitializerConfKey, StringUtils.join(",", targets)); - } - } - - private void startWebApp() throws IOException { - URI uri = URI.create("http://" + NetUtils.getHostPortString(bindAddress)); - - apiServer = new HttpServer2.Builder() - .setName("api-server") - .setConf(getConfig()) - .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()) - .setUsernameConfKey(RM_WEBAPP_SPNEGO_USER_NAME_KEY) - .setKeytabConfKey(RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) - .addEndpoint(uri).build(); - - String apiPackages = - ApiServer.class.getPackage().getName() + SEP - + GenericExceptionHandler.class.getPackage().getName() + SEP - + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); - apiServer.addJerseyResourcePackage(apiPackages, "/*"); - - try { - logger.info("Service starting up. Logging start..."); - apiServer.start(); - logger.info("Server status = {}", apiServer.toString()); - for (Configuration conf : apiServer.getWebAppContext() - .getConfigurations()) { - logger.info("Configurations = {}", conf); - } - logger.info("Context Path = {}", Collections.singletonList( - apiServer.getWebAppContext().getContextPath())); - logger.info("ResourceBase = {}", Collections.singletonList( - apiServer.getWebAppContext().getResourceBase())); - logger.info("War = {}", Collections - .singletonList(apiServer.getWebAppContext().getWar())); - } catch (Exception ex) { - logger.error("Hadoop HttpServer2 App **failed**", ex); - throw ex; - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md deleted file mode 100644 index 00b21dd..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md +++ /dev/null @@ -1,245 +0,0 @@ - - -## Examples - -### Create a simple single-component service with most attribute values as defaults -POST URL - http://localhost:9191/ws/v1/services - -##### POST Request JSON -```json -{ - "name": "hello-world", - "components" : - [ - { - "name": "hello", - "number_of_containers": 1, - "artifact": { - "id": "nginx:latest", - "type": "DOCKER" - }, - "launch_command": "./start_nginx.sh", - "resource": { - "cpus": 1, - "memory": "256" - } - } - ] -} -``` - -##### GET Response JSON -GET URL - http://localhost:9191/ws/v1/services/hello-world - -Note, lifetime value of -1 means unlimited lifetime. - -```json -{ - "name": "hello-world", - "id": "application_1503963985568_0002", - "lifetime": -1, - "components": [ - { - "name": "hello", - "dependencies": [], - "resource": { - "cpus": 1, - "memory": "256" - }, - "configuration": { - "properties": {}, - "env": {}, - "files": [] - }, - "quicklinks": [], - "containers": [ - { - "id": "container_e03_1503963985568_0002_01_000001", - "ip": "10.22.8.143", - "hostname": "myhost.local", - "state": "READY", - "launch_time": 1504051512412, - "bare_host": "10.22.8.143", - "component_name": "hello-0" - }, - { - "id": "container_e03_1503963985568_0002_01_000002", - "ip": "10.22.8.143", - "hostname": "myhost.local", - "state": "READY", - "launch_time": 1504051536450, - "bare_host": "10.22.8.143", - "component_name": "hello-1" - } - ], - "launch_command": "./start_nginx.sh", - "number_of_containers": 1, - "run_privileged_container": false - } - ], - "configuration": { - "properties": {}, - "env": {}, - "files": [] - }, - "quicklinks": {} -} - -``` -### Update to modify the lifetime of a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world - -##### PUT Request JSON - -Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below. - -```json -{ - "lifetime": 3600 -} -``` -### Stop a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world - -##### PUT Request JSON -```json -{ - "state": "STOPPED" -} -``` - -### Start a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world - -##### PUT Request JSON -```json -{ - "state": "STARTED" -} -``` - -### Update to flex up/down the no of containers (instances) of a component of a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello - -##### PUT Request JSON -```json -{ - "name": "hello", - "number_of_containers": 3 -} -``` - -### Destroy a service -DELETE URL - http://localhost:9191/ws/v1/services/hello-world - -*** - -### Create a complicated service - HBase -POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 - -##### POST Request JSON - -```json -{ - "name": "hbase-app-1", - "lifetime": "3600", - "components": [ - { - "name": "hbasemaster", - "number_of_containers": 1, - "artifact": { - "id": "hbase:latest", - "type": "DOCKER" - }, - "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start", - "resource": { - "cpus": 1, - "memory": "2048" - }, - "configuration": { - "env": { - "HBASE_LOG_DIR": "" - }, - "files": [ - { - "type": "XML", - "dest_file": "/etc/hadoop/conf/core-site.xml", - "properties": { - "fs.defaultFS": "${CLUSTER_FS_URI}" - } - }, - { - "type": "XML", - "dest_file": "/etc/hbase/conf/hbase-site.xml", - "properties": { - "hbase.cluster.distributed": "true", - "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}", - "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase", - "zookeeper.znode.parent": "${SERVICE_ZK_PATH}", - "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}", - "hbase.master.info.port": "16010" - } - } - ] - } - }, - { - "name": "regionserver", - "number_of_containers": 3, - "unique_component_support": "true", - "artifact": { - "id": "hbase:latest", - "type": "DOCKER" - }, - "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start", - "resource": { - "cpus": 1, - "memory": "2048" - }, - "configuration": { - "env": { - "HBASE_LOG_DIR": "" - }, - "files": [ - { - "type": "XML", - "dest_file": "/etc/hadoop/conf/core-site.xml", - "properties": { - "fs.defaultFS": "${CLUSTER_FS_URI}" - } - }, - { - "type": "XML", - "dest_file": "/etc/hbase/conf/hbase-site.xml", - "properties": { - "hbase.cluster.distributed": "true", - "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}", - "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase", - "zookeeper.znode.parent": "${SERVICE_ZK_PATH}", - "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}", - "hbase.master.info.port": "16010", - "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}" - } - } - ] - } - } - ], - "quicklinks": { - "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status", - "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/" - } -} -``` diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml deleted file mode 100644 index 088b50c..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ /dev/null @@ -1,471 +0,0 @@ -# Hadoop YARN REST APIs for services v1 spec in YAML - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -swagger: '2.0' -info: - title: "YARN Simplified API layer for services" - description: | - Bringing a new service on YARN today is not a simple experience. The APIs of existing - frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) - or writing a complex spec (for declarative frameworks). - - This simplified REST API can be used to create and manage the lifecycle of YARN services. - In most cases, the application owner will not be forced to make any changes to their applications. - This is primarily true if the application is packaged with containerization technologies like Docker. - - This document describes the API specifications (aka. YarnFile) for deploying/managing - containerized services on YARN. The same JSON spec can be used for both REST API - and CLI to manage the services. - - version: "1.0.0" - license: - name: Apache 2.0 - url: http://www.apache.org/licenses/LICENSE-2.0.html -# the domain of the service -host: host.mycompany.com -port: 9191(default) -# array of all schemes that your API supports -schemes: - - http -consumes: - - application/json -produces: - - application/json -paths: - /ws/v1/services/version: - get: - summary: Get current version of the API server. - description: Get current version of the API server. - responses: - 200: - description: Successful request - - /ws/v1/services: - get: - summary: (TBD) List of services running in the cluster. - description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name. - responses: - 200: - description: An array of services - schema: - type: array - items: - $ref: '#/definitions/Service' - default: - description: Unexpected error - schema: - $ref: '#/definitions/ServiceStatus' - post: - summary: Create a service - description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state. - parameters: - - name: Service - in: body - description: Service request object - required: true - schema: - $ref: '#/definitions/Service' - responses: - 202: - description: The request to create a service is accepted - 400: - description: Invalid service definition provided in the request body - 500: - description: Failed to create a service - default: - description: Unexpected error - schema: - $ref: '#/definitions/ServiceStatus' - - /ws/v1/services/{service_name}: - put: - summary: Update a service or upgrade the binary version of the components of a running service - description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. - The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD). - parameters: - - name: service_name - in: path - description: Service name - required: true - type: string - - name: Service - in: body - description: The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service - required: true - schema: - $ref: '#/definitions/Service' - responses: - 204: - description: Update or upgrade was successful - 404: - description: Service does not exist - default: - description: Unexpected error - schema: - $ref: '#/definitions/ServiceStatus' - delete: - summary: Destroy a service - description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc. - parameters: - - name: service_name - in: path - description: Service name - required: true - type: string - responses: - 204: - description: Destroy was successful - 404: - description: Service does not exist - default: - description: Unexpected error - schema: - $ref: '#/definitions/ServiceStatus' - get: - summary: Get details of a service. - description: Return the details (including containers) of a running service - parameters: - - name: service_name - in: path - description: Service name - required: true - type: string - responses: - 200: - description: a service object - schema: - type: object - items: - $ref: '#/definitions/Service' - examples: - service_name: logsearch - artifact: - id: logsearch:latest - type: docker - 404: - description: Service does not exist - default: - description: Unexpected error - schema: - $ref: '#/definitions/ServiceStatus' - /ws/v1/services/{service_name}/components/{component_name}: - put: - summary: Flex a component's number of instances. - description: Set a component's desired number of instanes - parameters: - - name: service_name - in: path - description: Service name - required: true - type: string - - name: component_name - in: path - description: Component name - required: true - type: string - - name: Component - in: body - description: The definition of a component which contains the updated number of instances. - required: true - schema: - $ref: '#/definitions/Component' - responses: - 200: - description: Flex was successful - 404: - description: Service does not exist - default: - description: Unexpected error - schema: - $ref: '#/definitions/ServiceStatus' -definitions: - Service: - description: a service resource has the following attributes. - required: - - name - properties: - name: - type: string - description: A unique service name. If Registry DNS is enabled, the max length is 63 characters. - id: - type: string - description: A unique service id. - artifact: - description: The default artifact for all components of the service except the components which has Artifact type set to SERVICE (optional). - $ref: '#/definitions/Artifact' - resource: - description: The default resource for all components of the service (optional). - $ref: '#/definitions/Resource' - launch_time: - type: string - format: date - description: The time when the service was created, e.g. 2016-03-16T01:01:49.000Z. - number_of_running_containers: - type: integer - format: int64 - description: In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests. - lifetime: - type: integer - format: int64 - description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value. - placement_policy: - description: (TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902. - $ref: '#/definitions/PlacementPolicy' - components: - description: Components of a service. - type: array - items: - $ref: '#/definitions/Component' - configuration: - description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level. - $ref: '#/definitions/Configuration' - state: - description: State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state. - $ref: '#/definitions/ServiceState' - quicklinks: - type: object - description: A blob of key-value pairs of quicklinks to be exported for a service. - additionalProperties: - type: string - queue: - type: string - description: The YARN queue that this service should be submitted to. - Resource: - description: - Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise. - properties: - profile: - type: string - description: Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc. - cpus: - type: integer - format: int32 - description: Amount of vcores allocated to each container (optional but overrides cpus in profile if specified). - memory: - type: string - description: Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB. - PlacementPolicy: - description: Placement policy of an instance of a service. This feature is in the works in YARN-6592. - properties: - label: - type: string - description: Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc. - Artifact: - description: Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized. - required: - - id - properties: - id: - type: string - description: Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc. - type: - type: string - description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored). - enum: - - DOCKER - - TARBALL - - SERVICE - default: DOCKER - uri: - type: string - description: Artifact location to support multiple artifact stores (optional). - Component: - description: One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services. - required: - - name - properties: - name: - type: string - description: Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters. - state: - description: The state of the component - $ref: "#/definitions/ComponentState" - dependencies: - type: array - items: - type: string - description: An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG. - readiness_check: - description: Readiness check for this component. - $ref: '#/definitions/ReadinessCheck' - artifact: - description: Artifact of the component (optional). If not specified, the service level global artifact takes effect. - $ref: '#/definitions/Artifact' - launch_command: - type: string - description: The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any). - resource: - description: Resource of this component (optional). If not specified, the service level global resource takes effect. - $ref: '#/definitions/Resource' - number_of_containers: - type: integer - format: int64 - description: Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect. - run_privileged_container: - type: boolean - description: Run all containers of this component in privileged mode (YARN-4262). - placement_policy: - description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details. - $ref: '#/definitions/PlacementPolicy' - configuration: - description: Config properties for this component. - $ref: '#/definitions/Configuration' - quicklinks: - type: array - items: - type: string - description: A list of quicklink keys defined at the service level, and to be resolved by this component. - ReadinessCheck: - description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases. - required: - - type - properties: - type: - type: string - description: E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content). - enum: - - HTTP - - PORT - properties: - type: object - description: A blob of key value pairs that will be used to configure the check. - additionalProperties: - type: string - artifact: - description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET - $ref: '#/definitions/Artifact' - Configuration: - description: Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported. - properties: - properties: - type: object - description: A blob of key-value pairs for configuring the YARN service AM - additionalProperties: - type: string - env: - type: object - description: A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection. - additionalProperties: - type: string - files: - description: Array of list of files that needs to be created and made available as volumes in the service component containers. - type: array - items: - $ref: '#/definitions/ConfigFile' - ConfigFile: - description: A config file that needs to be created and made available as a volume in a service component container. - properties: - type: - type: string - description: Config file in the standard format like xml, properties, json, yaml, template. - enum: - - XML - - PROPERTIES - - JSON - - YAML - - TEMPLATE - - ENV - - HADOOP_XML - dest_file: - type: string - description: The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf. - src_file: - type: string - description: This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported. - properties: - type: object - description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file. - Container: - description: An instance of a running service container. - properties: - id: - type: string - description: Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002. - launch_time: - type: string - format: date - description: The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time. - ip: - type: string - description: IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007. - hostname: - type: string - description: Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007. - bare_host: - type: string - description: The bare node or host in which the container is running, e.g. cn008.example.com. - state: - description: State of the container of a service. - $ref: '#/definitions/ContainerState' - component_instance_name: - type: string - description: Name of the component instance that this container instance belongs to. Component instance name is named as $COMPONENT_NAME-i, where i is a - monotonically increasing integer. E.g. A componet called nginx can have multiple component instances named as nginx-0, nginx-1 etc. - Each component instance is backed by a container instance. - resource: - description: Resource used for this container. - $ref: '#/definitions/Resource' - artifact: - description: Artifact used for this container. - $ref: '#/definitions/Artifact' - privileged_container: - type: boolean - description: Container running in privileged mode or not. - ServiceState: - description: The current state of a service. - properties: - state: - type: string - description: enum of the state of the service - enum: - - ACCEPTED - - STARTED - - STABLE - - STOPPED - - FAILED - ContainerState: - description: The current state of the container of a service. - properties: - state: - type: string - description: enum of the state of the container - enum: - - INIT - - STARTED - - READY - ComponentState: - description: The state of the component - properties: - state: - type: string - description: enum of the state of the component - enum: - - FLEXING - - STABLE - ServiceStatus: - description: The current status of a submitted service, returned as a response to the GET API. - properties: - diagnostics: - type: string - description: Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state. - state: - description: Service state. - $ref: '#/definitions/ServiceState' - code: - type: integer - format: int32 - description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties deleted file mode 100644 index 8c679b9..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This is the log4j configuration for YARN Services REST API Server - -# Log rotation based on size (100KB) with a max of 10 backup files -log4j.rootLogger=INFO, restservicelog -log4j.threshhold=ALL - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n - -log4j.appender.restservicelog=org.apache.log4j.RollingFileAppender -log4j.appender.restservicelog.layout=org.apache.log4j.PatternLayout -log4j.appender.restservicelog.File=${REST_SERVICE_LOG_DIR}/restservice.log -log4j.appender.restservicelog.MaxFileSize=1GB -log4j.appender.restservicelog.MaxBackupIndex=10 - -# log layout skips stack-trace creation operations by avoiding line numbers and method -log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n - -# debug edition is much more expensive -#log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n - -# configure stderr -# set the conversion pattern of stderr -# Print the date in ISO 8601 format -log4j.appender.stderr=org.apache.log4j.ConsoleAppender -log4j.appender.stderr.Target=System.err -log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n - -log4j.appender.subprocess=org.apache.log4j.ConsoleAppender -log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout -log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n - -# for debugging REST API Service -#log4j.logger.org.apache.hadoop.yarn.services=DEBUG - -# uncomment to debug service lifecycle issues -#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG -#log4j.logger.org.apache.hadoop.yarn.service=DEBUG - -# uncomment for YARN operations -#log4j.logger.org.apache.hadoop.yarn.client=DEBUG - -# uncomment this to debug security problems -#log4j.logger.org.apache.hadoop.security=DEBUG - -#crank back on some noise -log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR -log4j.logger.org.apache.hadoop.hdfs=WARN -log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR - -log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN -log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN -log4j.logger.org.apache.zookeeper=WARN -log4j.logger.org.apache.curator.framework.state=ERROR -log4j.logger.org.apache.curator.framework.imps=WARN - -log4j.logger.org.mortbay.log=DEBUG diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app deleted file mode 100644 index 6a077b1..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -DON'T DELETE. REST WEBAPP RUN SCRIPT WILL STOP WORKING. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml deleted file mode 100644 index 1282c9f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - Jersey REST API - com.sun.jersey.spi.container.servlet.ServletContainer - - com.sun.jersey.config.property.packages - org.apache.hadoop.yarn.service.webapp,org.apache.hadoop.yarn.service.api,org.apache.hadoop.yarn.service.api.records - - - com.sun.jersey.api.json.POJOMappingFeature - true - - 1 - - - Jersey REST API - /* - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java deleted file mode 100644 index 3e08c3a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.service.api.records.Service; -import org.apache.hadoop.yarn.service.client.ServiceClient; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; - -/** - * A mock version of ServiceClient - This class is design - * to simulate various error conditions that will happen - * when a consumer class calls ServiceClient. - */ -public class ServiceClientTest extends ServiceClient { - - private Configuration conf = new Configuration(); - - protected static void init() { - } - - public ServiceClientTest() { - super(); - } - - @Override - public Configuration getConfig() { - return conf; - } - - @Override - public ApplicationId actionCreate(Service service) { - String serviceName = service.getName(); - ServiceApiUtil.validateNameFormat(serviceName, getConfig()); - return ApplicationId.newInstance(System.currentTimeMillis(), 1); - } - - @Override - public Service getStatus(String appName) { - if (appName == null) { - throw new NullPointerException(); - } - if (appName.equals("jenkins")) { - return new Service(); - } else { - throw new IllegalArgumentException(); - } - } - - @Override - public int actionStart(String serviceName) - throws YarnException, IOException { - if (serviceName == null) { - throw new NullPointerException(); - } - if (serviceName.equals("jenkins")) { - return EXIT_SUCCESS; - } else { - throw new ApplicationNotFoundException(""); - } - } - - @Override - public int actionStop(String serviceName, boolean waitForAppStopped) - throws YarnException, IOException { - if (serviceName == null) { - throw new NullPointerException(); - } - if (serviceName.equals("jenkins")) { - return EXIT_SUCCESS; - } else { - throw new ApplicationNotFoundException(""); - } - } - - @Override - public int actionDestroy(String serviceName) { - if (serviceName == null) { - throw new NullPointerException(); - } - if (serviceName.equals("jenkins")) { - return EXIT_SUCCESS; - } else { - throw new IllegalArgumentException(); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java deleted file mode 100644 index 2b22474..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.service.api.records.Artifact; -import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum; -import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.api.records.Resource; -import org.apache.hadoop.yarn.service.api.records.Service; -import org.apache.hadoop.yarn.service.api.records.ServiceState; -import org.apache.hadoop.yarn.service.client.ServiceClient; -import org.apache.hadoop.yarn.service.webapp.ApiServer; -import javax.ws.rs.Path; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; - -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.*; - -/** - * Test case for ApiServer REST API. - * - */ -public class TestApiServer { - private ApiServer apiServer; - - @Before - public void setup() throws Exception { - ServiceClient mockServerClient = new ServiceClientTest(); - Configuration conf = new Configuration(); - conf.set("yarn.api-service.service.client.class", - ServiceClientTest.class.getName()); - ApiServer.setServiceClient(mockServerClient); - this.apiServer = new ApiServer(conf); - } - - @Test - public void testPathAnnotation() { - assertNotNull(this.apiServer.getClass().getAnnotation(Path.class)); - assertTrue("The controller has the annotation Path", - this.apiServer.getClass().isAnnotationPresent(Path.class)); - final Path path = this.apiServer.getClass() - .getAnnotation(Path.class); - assertEquals("The path has /ws/v1 annotation", path.value(), - "/ws/v1"); - } - - @Test - public void testGetVersion() { - final Response actual = apiServer.getVersion(); - assertEquals("Version number is", actual.getStatus(), - Response.ok().build().getStatus()); - } - - @Test - public void testBadCreateService() { - Service service = new Service(); - // Test for invalid argument - final Response actual = apiServer.createService(service); - assertEquals("Create service is ", actual.getStatus(), - Response.status(Status.BAD_REQUEST).build().getStatus()); - } - - @Test - public void testGoodCreateService() { - Service service = new Service(); - service.setName("jenkins"); - Artifact artifact = new Artifact(); - artifact.setType(TypeEnum.DOCKER); - artifact.setId("jenkins:latest"); - Resource resource = new Resource(); - resource.setCpus(1); - resource.setMemory("2048"); - List components = new ArrayList(); - Component c = new Component(); - c.setName("jenkins"); - c.setNumberOfContainers(1L); - c.setArtifact(artifact); - c.setLaunchCommand(""); - c.setResource(resource); - components.add(c); - service.setComponents(components); - final Response actual = apiServer.createService(service); - assertEquals("Create service is ", actual.getStatus(), - Response.status(Status.ACCEPTED).build().getStatus()); - } - - @Test - public void testBadGetService() { - final Response actual = apiServer.getService("no-jenkins"); - assertEquals("Get service is ", actual.getStatus(), - Response.status(Status.NOT_FOUND).build().getStatus()); - } - - @Test - public void testBadGetService2() { - final Response actual = apiServer.getService(null); - assertEquals("Get service is ", actual.getStatus(), - Response.status(Status.INTERNAL_SERVER_ERROR) - .build().getStatus()); - } - - @Test - public void testGoodGetService() { - final Response actual = apiServer.getService("jenkins"); - assertEquals("Get service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); - } - - @Test - public void testBadDeleteService() { - final Response actual = apiServer.deleteService("no-jenkins"); - assertEquals("Delete service is ", actual.getStatus(), - Response.status(Status.BAD_REQUEST).build().getStatus()); - } - - @Test - public void testBadDeleteService2() { - final Response actual = apiServer.deleteService(null); - assertEquals("Delete service is ", actual.getStatus(), - Response.status(Status.INTERNAL_SERVER_ERROR) - .build().getStatus()); - } - - @Test - public void testGoodDeleteService() { - final Response actual = apiServer.deleteService("jenkins"); - assertEquals("Delete service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); - } - - @Test - public void testDecreaseContainerAndStop() { - Service service = new Service(); - service.setState(ServiceState.STOPPED); - service.setName("jenkins"); - Artifact artifact = new Artifact(); - artifact.setType(TypeEnum.DOCKER); - artifact.setId("jenkins:latest"); - Resource resource = new Resource(); - resource.setCpus(1); - resource.setMemory("2048"); - List components = new ArrayList(); - Component c = new Component(); - c.setName("jenkins"); - c.setNumberOfContainers(0L); - c.setArtifact(artifact); - c.setLaunchCommand(""); - c.setResource(resource); - components.add(c); - service.setComponents(components); - final Response actual = apiServer.updateService("jenkins", - service); - assertEquals("update service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); - } - - @Test - public void testBadDecreaseContainerAndStop() { - Service service = new Service(); - service.setState(ServiceState.STOPPED); - service.setName("no-jenkins"); - Artifact artifact = new Artifact(); - artifact.setType(TypeEnum.DOCKER); - artifact.setId("jenkins:latest"); - Resource resource = new Resource(); - resource.setCpus(1); - resource.setMemory("2048"); - List components = new ArrayList(); - Component c = new Component(); - c.setName("no-jenkins"); - c.setNumberOfContainers(-1L); - c.setArtifact(artifact); - c.setLaunchCommand(""); - c.setResource(resource); - components.add(c); - service.setComponents(components); - System.out.println("before stop"); - final Response actual = apiServer.updateService("no-jenkins", - service); - assertEquals("flex service is ", actual.getStatus(), - Response.status(Status.BAD_REQUEST).build().getStatus()); - } - - @Test - public void testIncreaseContainersAndStart() { - Service service = new Service(); - service.setState(ServiceState.STARTED); - service.setName("jenkins"); - Artifact artifact = new Artifact(); - artifact.setType(TypeEnum.DOCKER); - artifact.setId("jenkins:latest"); - Resource resource = new Resource(); - resource.setCpus(1); - resource.setMemory("2048"); - List components = new ArrayList(); - Component c = new Component(); - c.setName("jenkins"); - c.setNumberOfContainers(2L); - c.setArtifact(artifact); - c.setLaunchCommand(""); - c.setResource(resource); - components.add(c); - service.setComponents(components); - final Response actual = apiServer.updateService("jenkins", - service); - assertEquals("flex service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); - } - - @Test - public void testBadStartServices() { - Service service = new Service(); - service.setState(ServiceState.STARTED); - service.setName("no-jenkins"); - Artifact artifact = new Artifact(); - artifact.setType(TypeEnum.DOCKER); - artifact.setId("jenkins:latest"); - Resource resource = new Resource(); - resource.setCpus(1); - resource.setMemory("2048"); - List components = new ArrayList(); - Component c = new Component(); - c.setName("jenkins"); - c.setNumberOfContainers(2L); - c.setArtifact(artifact); - c.setLaunchCommand(""); - c.setResource(resource); - components.add(c); - service.setComponents(components); - final Response actual = apiServer.updateService("no-jenkins", - service); - assertEquals("start service is ", actual.getStatus(), - Response.status(Status.INTERNAL_SERVER_ERROR).build() - .getStatus()); - } - - @Test - public void testGoodStartServices() { - Service service = new Service(); - service.setState(ServiceState.STARTED); - service.setName("jenkins"); - Artifact artifact = new Artifact(); - artifact.setType(TypeEnum.DOCKER); - artifact.setId("jenkins:latest"); - Resource resource = new Resource(); - resource.setCpus(1); - resource.setMemory("2048"); - List components = new ArrayList(); - Component c = new Component(); - c.setName("jenkins"); - c.setNumberOfContainers(2L); - c.setArtifact(artifact); - c.setLaunchCommand(""); - c.setResource(resource); - components.add(c); - service.setComponents(components); - final Response actual = apiServer.updateService("jenkins", - service); - assertEquals("start service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); - } - - @Test - public void testBadStopServices() { - Service service = new Service(); - service.setState(ServiceState.STOPPED); - service.setName("no-jenkins"); - Artifact artifact = new Artifact(); - artifact.setType(TypeEnum.DOCKER); - artifact.setId("jenkins:latest"); - Resource resource = new Resource(); - resource.setCpus(1); - resource.setMemory("2048"); - List components = new ArrayList(); - Component c = new Component(); - c.setName("no-jenkins"); - c.setNumberOfContainers(-1L); - c.setArtifact(artifact); - c.setLaunchCommand(""); - c.setResource(resource); - components.add(c); - service.setComponents(components); - System.out.println("before stop"); - final Response actual = apiServer.updateService("no-jenkins", - service); - assertEquals("stop service is ", actual.getStatus(), - Response.status(Status.BAD_REQUEST).build().getStatus()); - } - - @Test - public void testGoodStopServices() { - Service service = new Service(); - service.setState(ServiceState.STARTED); - service.setName("jenkins"); - Artifact artifact = new Artifact(); - artifact.setType(TypeEnum.DOCKER); - artifact.setId("jenkins:latest"); - Resource resource = new Resource(); - resource.setCpus(1); - resource.setMemory("2048"); - List components = new ArrayList(); - Component c = new Component(); - c.setName("jenkins"); - c.setNumberOfContainers(-1L); - c.setArtifact(artifact); - c.setLaunchCommand(""); - c.setResource(resource); - components.add(c); - service.setComponents(components); - System.out.println("before stop"); - final Response actual = apiServer.updateService("jenkins", - service); - assertEquals("stop service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); - } - - @Test - public void testUpdateService() { - Service service = new Service(); - service.setState(ServiceState.STARTED); - service.setName("no-jenkins"); - Artifact artifact = new Artifact(); - artifact.setType(TypeEnum.DOCKER); - artifact.setId("jenkins:latest"); - Resource resource = new Resource(); - resource.setCpus(1); - resource.setMemory("2048"); - List components = new ArrayList(); - Component c = new Component(); - c.setName("no-jenkins"); - c.setNumberOfContainers(-1L); - c.setArtifact(artifact); - c.setLaunchCommand(""); - c.setResource(resource); - components.add(c); - service.setComponents(components); - System.out.println("before stop"); - final Response actual = apiServer.updateService("no-jenkins", - service); - assertEquals("update service is ", actual.getStatus(), - Response.status(Status.INTERNAL_SERVER_ERROR) - .build().getStatus()); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml new file mode 100644 index 0000000..b89146a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml @@ -0,0 +1,20 @@ + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml new file mode 100644 index 0000000..ddea2a1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml @@ -0,0 +1,130 @@ + + + 4.0.0 + + org.apache.hadoop + hadoop-yarn-applications + 3.1.0-SNAPSHOT + + hadoop-yarn-services-api + Apache Hadoop YARN Services API + jar + Hadoop YARN REST APIs for services + + + + + + + src/main/resources + true + + + src/main/scripts/ + true + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + + + development + ${project.url} + + + + + + + + + + test-jar + + + + + + + + + + + + + + org.apache.hadoop + hadoop-yarn-services-core + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-common + + + org.slf4j + slf4j-api + + + org.eclipse.jetty + jetty-webapp + + + com.google.inject + guice + + + javax.ws.rs + jsr311-api + + + org.mockito + mockito-all + test + + + + + + + + org.apache.hadoop + hadoop-common + test-jar + + + junit + junit + test + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java new file mode 100644 index 0000000..1bb6c93 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.webapp; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.api.records.ServiceStatus; +import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED; +import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; + +/** + * The rest API endpoints for users to manage services on YARN. + */ +@Singleton +@Path(CONTEXT_ROOT) +public class ApiServer { + + public ApiServer() { + super(); + } + + @Inject + public ApiServer(Configuration conf) { + super(); + } + + private static final Logger LOG = + LoggerFactory.getLogger(ApiServer.class); + private static Configuration YARN_CONFIG = new YarnConfiguration(); + private static ServiceClient SERVICE_CLIENT; + + static { + init(); + } + + // initialize all the common resources - order is important + private static void init() { + SERVICE_CLIENT = new ServiceClient(); + SERVICE_CLIENT.init(YARN_CONFIG); + SERVICE_CLIENT.start(); + } + + @GET + @Path(VERSION) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response getVersion() { + String version = VersionInfo.getBuildVersion(); + LOG.info(version); + return Response.ok("{ \"hadoop_version\": \"" + version + "\"}").build(); + } + + @POST + @Path(SERVICE_ROOT_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response createService(Service service) { + LOG.info("POST: createService = {}", service); + ServiceStatus serviceStatus = new ServiceStatus(); + try { + ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service); + LOG.info("Successfully created service " + service.getName() + + " applicationId = " + applicationId); + serviceStatus.setState(ACCEPTED); + serviceStatus.setUri( + CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service + .getName()); + return Response.status(Status.ACCEPTED).entity(serviceStatus).build(); + } catch (IllegalArgumentException e) { + serviceStatus.setDiagnostics(e.getMessage()); + return Response.status(Status.BAD_REQUEST).entity(serviceStatus) + .build(); + } catch (Exception e) { + String message = "Failed to create service " + service.getName(); + LOG.error(message, e); + serviceStatus.setDiagnostics(message + ": " + e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(serviceStatus).build(); + } + } + + @GET + @Path(SERVICE_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response getService(@PathParam(SERVICE_NAME) String appName) { + LOG.info("GET: getService for appName = {}", appName); + ServiceStatus serviceStatus = new ServiceStatus(); + try { + Service app = SERVICE_CLIENT.getStatus(appName); + return Response.ok(app).build(); + } catch (IllegalArgumentException e) { + serviceStatus.setDiagnostics(e.getMessage()); + serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID); + return Response.status(Status.NOT_FOUND).entity(serviceStatus) + .build(); + } catch (Exception e) { + LOG.error("Get service failed", e); + serviceStatus + .setDiagnostics("Failed to retrieve service: " + e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(serviceStatus).build(); + } + } + + @DELETE + @Path(SERVICE_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response deleteService(@PathParam(SERVICE_NAME) String appName) { + LOG.info("DELETE: deleteService for appName = {}", appName); + return stopService(appName, true); + } + + private Response stopService(String appName, boolean destroy) { + try { + SERVICE_CLIENT.actionStop(appName, destroy); + if (destroy) { + SERVICE_CLIENT.actionDestroy(appName); + LOG.info("Successfully deleted service {}", appName); + } else { + LOG.info("Successfully stopped service {}", appName); + } + return Response.status(Status.OK).build(); + } catch (ApplicationNotFoundException e) { + ServiceStatus serviceStatus = new ServiceStatus(); + serviceStatus.setDiagnostics( + "Service " + appName + " is not found in YARN: " + e.getMessage()); + return Response.status(Status.BAD_REQUEST).entity(serviceStatus) + .build(); + } catch (Exception e) { + ServiceStatus serviceStatus = new ServiceStatus(); + serviceStatus.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(serviceStatus).build(); + } + } + + @PUT + @Path(COMPONENT_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN }) + public Response updateComponent(@PathParam(SERVICE_NAME) String appName, + @PathParam(COMPONENT_NAME) String componentName, Component component) { + + if (component.getNumberOfContainers() < 0) { + return Response.status(Status.BAD_REQUEST).entity( + "Service = " + appName + ", Component = " + component.getName() + + ": Invalid number of containers specified " + component + .getNumberOfContainers()).build(); + } + ServiceStatus status = new ServiceStatus(); + try { + Map original = SERVICE_CLIENT.flexByRestService(appName, + Collections.singletonMap(component.getName(), + component.getNumberOfContainers())); + status.setDiagnostics( + "Updating component (" + componentName + ") size from " + original + .get(componentName) + " to " + component.getNumberOfContainers()); + return Response.ok().entity(status).build(); + } catch (YarnException | IOException e) { + status.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status) + .build(); + } + } + + @PUT + @Path(SERVICE_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response updateService(@PathParam(SERVICE_NAME) String appName, + Service updateServiceData) { + LOG.info("PUT: updateService for app = {} with data = {}", appName, + updateServiceData); + + // Ignore the app name provided in updateServiceData and always use appName + // path param + updateServiceData.setName(appName); + + // For STOP the app should be running. If already stopped then this + // operation will be a no-op. For START it should be in stopped state. + // If already running then this operation will be a no-op. + if (updateServiceData.getState() != null + && updateServiceData.getState() == ServiceState.STOPPED) { + return stopService(appName, false); + } + + // If a START is requested + if (updateServiceData.getState() != null + && updateServiceData.getState() == ServiceState.STARTED) { + return startService(appName); + } + + // If new lifetime value specified then update it + if (updateServiceData.getLifetime() != null + && updateServiceData.getLifetime() > 0) { + return updateLifetime(appName, updateServiceData); + } + + // If nothing happens consider it a no-op + return Response.status(Status.NO_CONTENT).build(); + } + + private Response updateLifetime(String appName, Service updateAppData) { + ServiceStatus status = new ServiceStatus(); + try { + String newLifeTime = + SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); + status.setDiagnostics( + "Service (" + appName + ")'s lifeTime is updated to " + newLifeTime + + ", " + updateAppData.getLifetime() + + " seconds remaining"); + return Response.ok(status).build(); + } catch (Exception e) { + String message = + "Failed to update service (" + appName + ")'s lifetime to " + + updateAppData.getLifetime(); + LOG.error(message, e); + status.setDiagnostics(message + ": " + e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status) + .build(); + } + } + + private Response startService(String appName) { + ServiceStatus status = new ServiceStatus(); + try { + SERVICE_CLIENT.actionStart(appName); + LOG.info("Successfully started service " + appName); + status.setDiagnostics("Service " + appName + " is successfully started."); + status.setState(ServiceState.ACCEPTED); + return Response.ok(status).build(); + } catch (Exception e) { + String message = "Failed to start service " + appName; + status.setDiagnostics(message + ": " + e.getMessage()); + LOG.info(message, e); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(status).build(); + } + } + + /** + * Used by negative test case. + * + * @param mockServerClient - A mocked version of ServiceClient + */ + public static void setServiceClient(ServiceClient mockServerClient) { + SERVICE_CLIENT = mockServerClient; + SERVICE_CLIENT.init(YARN_CONFIG); + SERVICE_CLIENT.start(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java new file mode 100644 index 0000000..f4acd94 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.webapp; + +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.AuthenticationFilterInitializer; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.eclipse.jetty.webapp.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*; + +/** + * This class launches the web service using Hadoop HttpServer2 (which uses + * an embedded Jetty container). This is the entry point to your service. + * The Java command used to launch this app should call the main method. + */ +public class ApiServerWebApp extends AbstractService { + private static final Logger logger = LoggerFactory + .getLogger(ApiServerWebApp.class); + private static final String SEP = ";"; + + // REST API server for YARN native services + private HttpServer2 apiServer; + private InetSocketAddress bindAddress; + + public static void main(String[] args) throws IOException { + ApiServerWebApp apiWebApp = new ApiServerWebApp(); + try { + apiWebApp.init(new YarnConfiguration()); + apiWebApp.serviceStart(); + } catch (Exception e) { + logger.error("Got exception starting", e); + apiWebApp.close(); + } + } + + public ApiServerWebApp() { + super(ApiServerWebApp.class.getName()); + } + + @Override + protected void serviceStart() throws Exception { + bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS, + DEFAULT_API_SERVER_ADDRESS, DEFAULT_API_SERVER_PORT); + logger.info("YARN API server running on " + bindAddress); + if (UserGroupInformation.isSecurityEnabled()) { + doSecureLogin(getConfig()); + } + startWebApp(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (apiServer != null) { + apiServer.stop(); + } + super.serviceStop(); + } + + private void doSecureLogin(org.apache.hadoop.conf.Configuration conf) + throws IOException { + SecurityUtil.login(conf, YarnConfiguration.RM_KEYTAB, + YarnConfiguration.RM_PRINCIPAL, bindAddress.getHostName()); + addFilters(conf); + } + + private void addFilters(org.apache.hadoop.conf.Configuration conf) { + // Always load pseudo authentication filter to parse "user.name" in an URL + // to identify a HTTP request's user. + boolean hasHadoopAuthFilterInitializer = false; + String filterInitializerConfKey = "hadoop.http.filter.initializers"; + Class[] initializersClasses = + conf.getClasses(filterInitializerConfKey); + List targets = new ArrayList(); + if (initializersClasses != null) { + for (Class initializer : initializersClasses) { + if (initializer.getName().equals( + AuthenticationFilterInitializer.class.getName())) { + hasHadoopAuthFilterInitializer = true; + break; + } + targets.add(initializer.getName()); + } + } + if (!hasHadoopAuthFilterInitializer) { + targets.add(AuthenticationFilterInitializer.class.getName()); + conf.set(filterInitializerConfKey, StringUtils.join(",", targets)); + } + } + + private void startWebApp() throws IOException { + URI uri = URI.create("http://" + NetUtils.getHostPortString(bindAddress)); + + apiServer = new HttpServer2.Builder() + .setName("api-server") + .setConf(getConfig()) + .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()) + .setUsernameConfKey(RM_WEBAPP_SPNEGO_USER_NAME_KEY) + .setKeytabConfKey(RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) + .addEndpoint(uri).build(); + + String apiPackages = + ApiServer.class.getPackage().getName() + SEP + + GenericExceptionHandler.class.getPackage().getName() + SEP + + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); + apiServer.addJerseyResourcePackage(apiPackages, "/*"); + + try { + logger.info("Service starting up. Logging start..."); + apiServer.start(); + logger.info("Server status = {}", apiServer.toString()); + for (Configuration conf : apiServer.getWebAppContext() + .getConfigurations()) { + logger.info("Configurations = {}", conf); + } + logger.info("Context Path = {}", Collections.singletonList( + apiServer.getWebAppContext().getContextPath())); + logger.info("ResourceBase = {}", Collections.singletonList( + apiServer.getWebAppContext().getResourceBase())); + logger.info("War = {}", Collections + .singletonList(apiServer.getWebAppContext().getWar())); + } catch (Exception ex) { + logger.error("Hadoop HttpServer2 App **failed**", ex); + throw ex; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md new file mode 100644 index 0000000..00b21dd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md @@ -0,0 +1,245 @@ + + +## Examples + +### Create a simple single-component service with most attribute values as defaults +POST URL - http://localhost:9191/ws/v1/services + +##### POST Request JSON +```json +{ + "name": "hello-world", + "components" : + [ + { + "name": "hello", + "number_of_containers": 1, + "artifact": { + "id": "nginx:latest", + "type": "DOCKER" + }, + "launch_command": "./start_nginx.sh", + "resource": { + "cpus": 1, + "memory": "256" + } + } + ] +} +``` + +##### GET Response JSON +GET URL - http://localhost:9191/ws/v1/services/hello-world + +Note, lifetime value of -1 means unlimited lifetime. + +```json +{ + "name": "hello-world", + "id": "application_1503963985568_0002", + "lifetime": -1, + "components": [ + { + "name": "hello", + "dependencies": [], + "resource": { + "cpus": 1, + "memory": "256" + }, + "configuration": { + "properties": {}, + "env": {}, + "files": [] + }, + "quicklinks": [], + "containers": [ + { + "id": "container_e03_1503963985568_0002_01_000001", + "ip": "10.22.8.143", + "hostname": "myhost.local", + "state": "READY", + "launch_time": 1504051512412, + "bare_host": "10.22.8.143", + "component_name": "hello-0" + }, + { + "id": "container_e03_1503963985568_0002_01_000002", + "ip": "10.22.8.143", + "hostname": "myhost.local", + "state": "READY", + "launch_time": 1504051536450, + "bare_host": "10.22.8.143", + "component_name": "hello-1" + } + ], + "launch_command": "./start_nginx.sh", + "number_of_containers": 1, + "run_privileged_container": false + } + ], + "configuration": { + "properties": {}, + "env": {}, + "files": [] + }, + "quicklinks": {} +} + +``` +### Update to modify the lifetime of a service +PUT URL - http://localhost:9191/ws/v1/services/hello-world + +##### PUT Request JSON + +Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below. + +```json +{ + "lifetime": 3600 +} +``` +### Stop a service +PUT URL - http://localhost:9191/ws/v1/services/hello-world + +##### PUT Request JSON +```json +{ + "state": "STOPPED" +} +``` + +### Start a service +PUT URL - http://localhost:9191/ws/v1/services/hello-world + +##### PUT Request JSON +```json +{ + "state": "STARTED" +} +``` + +### Update to flex up/down the no of containers (instances) of a component of a service +PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello + +##### PUT Request JSON +```json +{ + "name": "hello", + "number_of_containers": 3 +} +``` + +### Destroy a service +DELETE URL - http://localhost:9191/ws/v1/services/hello-world + +*** + +### Create a complicated service - HBase +POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 + +##### POST Request JSON + +```json +{ + "name": "hbase-app-1", + "lifetime": "3600", + "components": [ + { + "name": "hbasemaster", + "number_of_containers": 1, + "artifact": { + "id": "hbase:latest", + "type": "DOCKER" + }, + "launch_command": "/usr/hdp/current/hbase-master/bin/hbase master start", + "resource": { + "cpus": 1, + "memory": "2048" + }, + "configuration": { + "env": { + "HBASE_LOG_DIR": "" + }, + "files": [ + { + "type": "XML", + "dest_file": "/etc/hadoop/conf/core-site.xml", + "properties": { + "fs.defaultFS": "${CLUSTER_FS_URI}" + } + }, + { + "type": "XML", + "dest_file": "/etc/hbase/conf/hbase-site.xml", + "properties": { + "hbase.cluster.distributed": "true", + "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}", + "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase", + "zookeeper.znode.parent": "${SERVICE_ZK_PATH}", + "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}", + "hbase.master.info.port": "16010" + } + } + ] + } + }, + { + "name": "regionserver", + "number_of_containers": 3, + "unique_component_support": "true", + "artifact": { + "id": "hbase:latest", + "type": "DOCKER" + }, + "launch_command": "/usr/hdp/current/hbase-regionserver/bin/hbase regionserver start", + "resource": { + "cpus": 1, + "memory": "2048" + }, + "configuration": { + "env": { + "HBASE_LOG_DIR": "" + }, + "files": [ + { + "type": "XML", + "dest_file": "/etc/hadoop/conf/core-site.xml", + "properties": { + "fs.defaultFS": "${CLUSTER_FS_URI}" + } + }, + { + "type": "XML", + "dest_file": "/etc/hbase/conf/hbase-site.xml", + "properties": { + "hbase.cluster.distributed": "true", + "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}", + "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase", + "zookeeper.znode.parent": "${SERVICE_ZK_PATH}", + "hbase.master.hostname": "hbasemaster.${SERVICE_NAME}.${USER}.${DOMAIN}", + "hbase.master.info.port": "16010", + "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}" + } + } + ] + } + } + ], + "quicklinks": { + "HBase Master Status UI": "http://hbasemaster0.${SERVICE_NAME}.${USER}.${DOMAIN}:16010/master-status", + "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/" + } +} +``` diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml new file mode 100644 index 0000000..088b50c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -0,0 +1,471 @@ +# Hadoop YARN REST APIs for services v1 spec in YAML + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "YARN Simplified API layer for services" + description: | + Bringing a new service on YARN today is not a simple experience. The APIs of existing + frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) + or writing a complex spec (for declarative frameworks). + + This simplified REST API can be used to create and manage the lifecycle of YARN services. + In most cases, the application owner will not be forced to make any changes to their applications. + This is primarily true if the application is packaged with containerization technologies like Docker. + + This document describes the API specifications (aka. YarnFile) for deploying/managing + containerized services on YARN. The same JSON spec can be used for both REST API + and CLI to manage the services. + + version: "1.0.0" + license: + name: Apache 2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html +# the domain of the service +host: host.mycompany.com +port: 9191(default) +# array of all schemes that your API supports +schemes: + - http +consumes: + - application/json +produces: + - application/json +paths: + /ws/v1/services/version: + get: + summary: Get current version of the API server. + description: Get current version of the API server. + responses: + 200: + description: Successful request + + /ws/v1/services: + get: + summary: (TBD) List of services running in the cluster. + description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name. + responses: + 200: + description: An array of services + schema: + type: array + items: + $ref: '#/definitions/Service' + default: + description: Unexpected error + schema: + $ref: '#/definitions/ServiceStatus' + post: + summary: Create a service + description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state. + parameters: + - name: Service + in: body + description: Service request object + required: true + schema: + $ref: '#/definitions/Service' + responses: + 202: + description: The request to create a service is accepted + 400: + description: Invalid service definition provided in the request body + 500: + description: Failed to create a service + default: + description: Unexpected error + schema: + $ref: '#/definitions/ServiceStatus' + + /ws/v1/services/{service_name}: + put: + summary: Update a service or upgrade the binary version of the components of a running service + description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. + The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD). + parameters: + - name: service_name + in: path + description: Service name + required: true + type: string + - name: Service + in: body + description: The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service + required: true + schema: + $ref: '#/definitions/Service' + responses: + 204: + description: Update or upgrade was successful + 404: + description: Service does not exist + default: + description: Unexpected error + schema: + $ref: '#/definitions/ServiceStatus' + delete: + summary: Destroy a service + description: Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc. + parameters: + - name: service_name + in: path + description: Service name + required: true + type: string + responses: + 204: + description: Destroy was successful + 404: + description: Service does not exist + default: + description: Unexpected error + schema: + $ref: '#/definitions/ServiceStatus' + get: + summary: Get details of a service. + description: Return the details (including containers) of a running service + parameters: + - name: service_name + in: path + description: Service name + required: true + type: string + responses: + 200: + description: a service object + schema: + type: object + items: + $ref: '#/definitions/Service' + examples: + service_name: logsearch + artifact: + id: logsearch:latest + type: docker + 404: + description: Service does not exist + default: + description: Unexpected error + schema: + $ref: '#/definitions/ServiceStatus' + /ws/v1/services/{service_name}/components/{component_name}: + put: + summary: Flex a component's number of instances. + description: Set a component's desired number of instanes + parameters: + - name: service_name + in: path + description: Service name + required: true + type: string + - name: component_name + in: path + description: Component name + required: true + type: string + - name: Component + in: body + description: The definition of a component which contains the updated number of instances. + required: true + schema: + $ref: '#/definitions/Component' + responses: + 200: + description: Flex was successful + 404: + description: Service does not exist + default: + description: Unexpected error + schema: + $ref: '#/definitions/ServiceStatus' +definitions: + Service: + description: a service resource has the following attributes. + required: + - name + properties: + name: + type: string + description: A unique service name. If Registry DNS is enabled, the max length is 63 characters. + id: + type: string + description: A unique service id. + artifact: + description: The default artifact for all components of the service except the components which has Artifact type set to SERVICE (optional). + $ref: '#/definitions/Artifact' + resource: + description: The default resource for all components of the service (optional). + $ref: '#/definitions/Resource' + launch_time: + type: string + format: date + description: The time when the service was created, e.g. 2016-03-16T01:01:49.000Z. + number_of_running_containers: + type: integer + format: int64 + description: In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests. + lifetime: + type: integer + format: int64 + description: Life time (in seconds) of the service from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value. + placement_policy: + description: (TBD) Advanced scheduling and placement policies. If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902. + $ref: '#/definitions/PlacementPolicy' + components: + description: Components of a service. + type: array + items: + $ref: '#/definitions/Component' + configuration: + description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level. + $ref: '#/definitions/Configuration' + state: + description: State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state. + $ref: '#/definitions/ServiceState' + quicklinks: + type: object + description: A blob of key-value pairs of quicklinks to be exported for a service. + additionalProperties: + type: string + queue: + type: string + description: The YARN queue that this service should be submitted to. + Resource: + description: + Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or service. The resource specified at the service (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are expected. It raises a validation exception otherwise. + properties: + profile: + type: string + description: Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc. + cpus: + type: integer + format: int32 + description: Amount of vcores allocated to each container (optional but overrides cpus in profile if specified). + memory: + type: string + description: Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB. + PlacementPolicy: + description: Placement policy of an instance of a service. This feature is in the works in YARN-6592. + properties: + label: + type: string + description: Assigns a service to a named partition of the cluster where the service desires to run (optional). If not specified all services are submitted to a default label of the service owner. One or more labels can be setup for each service owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc. + Artifact: + description: Artifact of a service component. If not specified, component will just run the bare launch command and no artifact will be localized. + required: + - id + properties: + id: + type: string + description: Artifact id. Examples are package location uri for tarball based services, image name for docker, name of service, etc. + type: + type: string + description: Artifact type, like docker, tarball, etc. (optional). For TARBALL type, the specified tarball will be localized to the container local working directory under a folder named lib. For SERVICE type, the service specified will be read and its components will be added into this service. The original component with artifact type SERVICE will be removed (any properties specified in the original component will be ignored). + enum: + - DOCKER + - TARBALL + - SERVICE + default: DOCKER + uri: + type: string + description: Artifact location to support multiple artifact stores (optional). + Component: + description: One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services. + required: + - name + properties: + name: + type: string + description: Name of the service component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters. + state: + description: The state of the component + $ref: "#/definitions/ComponentState" + dependencies: + type: array + items: + type: string + description: An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of a service should be represented as a DAG. + readiness_check: + description: Readiness check for this component. + $ref: '#/definitions/ReadinessCheck' + artifact: + description: Artifact of the component (optional). If not specified, the service level global artifact takes effect. + $ref: '#/definitions/Artifact' + launch_command: + type: string + description: The custom launch command of this component (optional for DOCKER component, required otherwise). When specified at the component level, it overrides the value specified at the global level (if any). + resource: + description: Resource of this component (optional). If not specified, the service level global resource takes effect. + $ref: '#/definitions/Resource' + number_of_containers: + type: integer + format: int64 + description: Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect. + run_privileged_container: + type: boolean + description: Run all containers of this component in privileged mode (YARN-4262). + placement_policy: + description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details. + $ref: '#/definitions/PlacementPolicy' + configuration: + description: Config properties for this component. + $ref: '#/definitions/Configuration' + quicklinks: + type: array + items: + type: string + description: A list of quicklink keys defined at the service level, and to be resolved by this component. + ReadinessCheck: + description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases. + required: + - type + properties: + type: + type: string + description: E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content). + enum: + - HTTP + - PORT + properties: + type: object + description: A blob of key value pairs that will be used to configure the check. + additionalProperties: + type: string + artifact: + description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET + $ref: '#/definitions/Artifact' + Configuration: + description: Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported. + properties: + properties: + type: object + description: A blob of key-value pairs for configuring the YARN service AM + additionalProperties: + type: string + env: + type: object + description: A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection. + additionalProperties: + type: string + files: + description: Array of list of files that needs to be created and made available as volumes in the service component containers. + type: array + items: + $ref: '#/definitions/ConfigFile' + ConfigFile: + description: A config file that needs to be created and made available as a volume in a service component container. + properties: + type: + type: string + description: Config file in the standard format like xml, properties, json, yaml, template. + enum: + - XML + - PROPERTIES + - JSON + - YAML + - TEMPLATE + - ENV + - HADOOP_XML + dest_file: + type: string + description: The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf. + src_file: + type: string + description: This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported. + properties: + type: object + description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file. + Container: + description: An instance of a running service container. + properties: + id: + type: string + description: Unique container id of a running service, e.g. container_e3751_1458061340047_0008_01_000002. + launch_time: + type: string + format: date + description: The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time. + ip: + type: string + description: IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007. + hostname: + type: string + description: Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007. + bare_host: + type: string + description: The bare node or host in which the container is running, e.g. cn008.example.com. + state: + description: State of the container of a service. + $ref: '#/definitions/ContainerState' + component_instance_name: + type: string + description: Name of the component instance that this container instance belongs to. Component instance name is named as $COMPONENT_NAME-i, where i is a + monotonically increasing integer. E.g. A componet called nginx can have multiple component instances named as nginx-0, nginx-1 etc. + Each component instance is backed by a container instance. + resource: + description: Resource used for this container. + $ref: '#/definitions/Resource' + artifact: + description: Artifact used for this container. + $ref: '#/definitions/Artifact' + privileged_container: + type: boolean + description: Container running in privileged mode or not. + ServiceState: + description: The current state of a service. + properties: + state: + type: string + description: enum of the state of the service + enum: + - ACCEPTED + - STARTED + - STABLE + - STOPPED + - FAILED + ContainerState: + description: The current state of the container of a service. + properties: + state: + type: string + description: enum of the state of the container + enum: + - INIT + - STARTED + - READY + ComponentState: + description: The state of the component + properties: + state: + type: string + description: enum of the state of the component + enum: + - FLEXING + - STABLE + ServiceStatus: + description: The current status of a submitted service, returned as a response to the GET API. + properties: + diagnostics: + type: string + description: Diagnostic information (if any) for the reason of the current state of the service. It typically has a non-null value, if the service is in a non-running state. + state: + description: Service state. + $ref: '#/definitions/ServiceState' + code: + type: integer + format: int32 + description: An error code specific to a scenario which service owners should be able to use to understand the failure in addition to the diagnostic information. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/log4j-server.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/log4j-server.properties new file mode 100644 index 0000000..8c679b9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/log4j-server.properties @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is the log4j configuration for YARN Services REST API Server + +# Log rotation based on size (100KB) with a max of 10 backup files +log4j.rootLogger=INFO, restservicelog +log4j.threshhold=ALL + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n + +log4j.appender.restservicelog=org.apache.log4j.RollingFileAppender +log4j.appender.restservicelog.layout=org.apache.log4j.PatternLayout +log4j.appender.restservicelog.File=${REST_SERVICE_LOG_DIR}/restservice.log +log4j.appender.restservicelog.MaxFileSize=1GB +log4j.appender.restservicelog.MaxBackupIndex=10 + +# log layout skips stack-trace creation operations by avoiding line numbers and method +log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +# debug edition is much more expensive +#log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + +# configure stderr +# set the conversion pattern of stderr +# Print the date in ISO 8601 format +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +log4j.appender.subprocess=org.apache.log4j.ConsoleAppender +log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout +log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n + +# for debugging REST API Service +#log4j.logger.org.apache.hadoop.yarn.services=DEBUG + +# uncomment to debug service lifecycle issues +#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG +#log4j.logger.org.apache.hadoop.yarn.service=DEBUG + +# uncomment for YARN operations +#log4j.logger.org.apache.hadoop.yarn.client=DEBUG + +# uncomment this to debug security problems +#log4j.logger.org.apache.hadoop.security=DEBUG + +#crank back on some noise +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +log4j.logger.org.apache.hadoop.hdfs=WARN +log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR + +log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN +log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.curator.framework.state=ERROR +log4j.logger.org.apache.curator.framework.imps=WARN + +log4j.logger.org.mortbay.log=DEBUG diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app new file mode 100644 index 0000000..6a077b1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/webapps/api-server/app @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DON'T DELETE. REST WEBAPP RUN SCRIPT WILL STOP WORKING. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 0000000..1282c9f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,36 @@ + + + + + + Jersey REST API + com.sun.jersey.spi.container.servlet.ServletContainer + + com.sun.jersey.config.property.packages + org.apache.hadoop.yarn.service.webapp,org.apache.hadoop.yarn.service.api,org.apache.hadoop.yarn.service.api.records + + + com.sun.jersey.api.json.POJOMappingFeature + true + + 1 + + + Jersey REST API + /* + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java new file mode 100644 index 0000000..3e08c3a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; + +/** + * A mock version of ServiceClient - This class is design + * to simulate various error conditions that will happen + * when a consumer class calls ServiceClient. + */ +public class ServiceClientTest extends ServiceClient { + + private Configuration conf = new Configuration(); + + protected static void init() { + } + + public ServiceClientTest() { + super(); + } + + @Override + public Configuration getConfig() { + return conf; + } + + @Override + public ApplicationId actionCreate(Service service) { + String serviceName = service.getName(); + ServiceApiUtil.validateNameFormat(serviceName, getConfig()); + return ApplicationId.newInstance(System.currentTimeMillis(), 1); + } + + @Override + public Service getStatus(String appName) { + if (appName == null) { + throw new NullPointerException(); + } + if (appName.equals("jenkins")) { + return new Service(); + } else { + throw new IllegalArgumentException(); + } + } + + @Override + public int actionStart(String serviceName) + throws YarnException, IOException { + if (serviceName == null) { + throw new NullPointerException(); + } + if (serviceName.equals("jenkins")) { + return EXIT_SUCCESS; + } else { + throw new ApplicationNotFoundException(""); + } + } + + @Override + public int actionStop(String serviceName, boolean waitForAppStopped) + throws YarnException, IOException { + if (serviceName == null) { + throw new NullPointerException(); + } + if (serviceName.equals("jenkins")) { + return EXIT_SUCCESS; + } else { + throw new ApplicationNotFoundException(""); + } + } + + @Override + public int actionDestroy(String serviceName) { + if (serviceName == null) { + throw new NullPointerException(); + } + if (serviceName.equals("jenkins")) { + return EXIT_SUCCESS; + } else { + throw new IllegalArgumentException(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java new file mode 100644 index 0000000..2b22474 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.service.api.records.Artifact; +import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.webapp.ApiServer; +import javax.ws.rs.Path; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Test case for ApiServer REST API. + * + */ +public class TestApiServer { + private ApiServer apiServer; + + @Before + public void setup() throws Exception { + ServiceClient mockServerClient = new ServiceClientTest(); + Configuration conf = new Configuration(); + conf.set("yarn.api-service.service.client.class", + ServiceClientTest.class.getName()); + ApiServer.setServiceClient(mockServerClient); + this.apiServer = new ApiServer(conf); + } + + @Test + public void testPathAnnotation() { + assertNotNull(this.apiServer.getClass().getAnnotation(Path.class)); + assertTrue("The controller has the annotation Path", + this.apiServer.getClass().isAnnotationPresent(Path.class)); + final Path path = this.apiServer.getClass() + .getAnnotation(Path.class); + assertEquals("The path has /ws/v1 annotation", path.value(), + "/ws/v1"); + } + + @Test + public void testGetVersion() { + final Response actual = apiServer.getVersion(); + assertEquals("Version number is", actual.getStatus(), + Response.ok().build().getStatus()); + } + + @Test + public void testBadCreateService() { + Service service = new Service(); + // Test for invalid argument + final Response actual = apiServer.createService(service); + assertEquals("Create service is ", actual.getStatus(), + Response.status(Status.BAD_REQUEST).build().getStatus()); + } + + @Test + public void testGoodCreateService() { + Service service = new Service(); + service.setName("jenkins"); + Artifact artifact = new Artifact(); + artifact.setType(TypeEnum.DOCKER); + artifact.setId("jenkins:latest"); + Resource resource = new Resource(); + resource.setCpus(1); + resource.setMemory("2048"); + List components = new ArrayList(); + Component c = new Component(); + c.setName("jenkins"); + c.setNumberOfContainers(1L); + c.setArtifact(artifact); + c.setLaunchCommand(""); + c.setResource(resource); + components.add(c); + service.setComponents(components); + final Response actual = apiServer.createService(service); + assertEquals("Create service is ", actual.getStatus(), + Response.status(Status.ACCEPTED).build().getStatus()); + } + + @Test + public void testBadGetService() { + final Response actual = apiServer.getService("no-jenkins"); + assertEquals("Get service is ", actual.getStatus(), + Response.status(Status.NOT_FOUND).build().getStatus()); + } + + @Test + public void testBadGetService2() { + final Response actual = apiServer.getService(null); + assertEquals("Get service is ", actual.getStatus(), + Response.status(Status.INTERNAL_SERVER_ERROR) + .build().getStatus()); + } + + @Test + public void testGoodGetService() { + final Response actual = apiServer.getService("jenkins"); + assertEquals("Get service is ", actual.getStatus(), + Response.status(Status.OK).build().getStatus()); + } + + @Test + public void testBadDeleteService() { + final Response actual = apiServer.deleteService("no-jenkins"); + assertEquals("Delete service is ", actual.getStatus(), + Response.status(Status.BAD_REQUEST).build().getStatus()); + } + + @Test + public void testBadDeleteService2() { + final Response actual = apiServer.deleteService(null); + assertEquals("Delete service is ", actual.getStatus(), + Response.status(Status.INTERNAL_SERVER_ERROR) + .build().getStatus()); + } + + @Test + public void testGoodDeleteService() { + final Response actual = apiServer.deleteService("jenkins"); + assertEquals("Delete service is ", actual.getStatus(), + Response.status(Status.OK).build().getStatus()); + } + + @Test + public void testDecreaseContainerAndStop() { + Service service = new Service(); + service.setState(ServiceState.STOPPED); + service.setName("jenkins"); + Artifact artifact = new Artifact(); + artifact.setType(TypeEnum.DOCKER); + artifact.setId("jenkins:latest"); + Resource resource = new Resource(); + resource.setCpus(1); + resource.setMemory("2048"); + List components = new ArrayList(); + Component c = new Component(); + c.setName("jenkins"); + c.setNumberOfContainers(0L); + c.setArtifact(artifact); + c.setLaunchCommand(""); + c.setResource(resource); + components.add(c); + service.setComponents(components); + final Response actual = apiServer.updateService("jenkins", + service); + assertEquals("update service is ", actual.getStatus(), + Response.status(Status.OK).build().getStatus()); + } + + @Test + public void testBadDecreaseContainerAndStop() { + Service service = new Service(); + service.setState(ServiceState.STOPPED); + service.setName("no-jenkins"); + Artifact artifact = new Artifact(); + artifact.setType(TypeEnum.DOCKER); + artifact.setId("jenkins:latest"); + Resource resource = new Resource(); + resource.setCpus(1); + resource.setMemory("2048"); + List components = new ArrayList(); + Component c = new Component(); + c.setName("no-jenkins"); + c.setNumberOfContainers(-1L); + c.setArtifact(artifact); + c.setLaunchCommand(""); + c.setResource(resource); + components.add(c); + service.setComponents(components); + System.out.println("before stop"); + final Response actual = apiServer.updateService("no-jenkins", + service); + assertEquals("flex service is ", actual.getStatus(), + Response.status(Status.BAD_REQUEST).build().getStatus()); + } + + @Test + public void testIncreaseContainersAndStart() { + Service service = new Service(); + service.setState(ServiceState.STARTED); + service.setName("jenkins"); + Artifact artifact = new Artifact(); + artifact.setType(TypeEnum.DOCKER); + artifact.setId("jenkins:latest"); + Resource resource = new Resource(); + resource.setCpus(1); + resource.setMemory("2048"); + List components = new ArrayList(); + Component c = new Component(); + c.setName("jenkins"); + c.setNumberOfContainers(2L); + c.setArtifact(artifact); + c.setLaunchCommand(""); + c.setResource(resource); + components.add(c); + service.setComponents(components); + final Response actual = apiServer.updateService("jenkins", + service); + assertEquals("flex service is ", actual.getStatus(), + Response.status(Status.OK).build().getStatus()); + } + + @Test + public void testBadStartServices() { + Service service = new Service(); + service.setState(ServiceState.STARTED); + service.setName("no-jenkins"); + Artifact artifact = new Artifact(); + artifact.setType(TypeEnum.DOCKER); + artifact.setId("jenkins:latest"); + Resource resource = new Resource(); + resource.setCpus(1); + resource.setMemory("2048"); + List components = new ArrayList(); + Component c = new Component(); + c.setName("jenkins"); + c.setNumberOfContainers(2L); + c.setArtifact(artifact); + c.setLaunchCommand(""); + c.setResource(resource); + components.add(c); + service.setComponents(components); + final Response actual = apiServer.updateService("no-jenkins", + service); + assertEquals("start service is ", actual.getStatus(), + Response.status(Status.INTERNAL_SERVER_ERROR).build() + .getStatus()); + } + + @Test + public void testGoodStartServices() { + Service service = new Service(); + service.setState(ServiceState.STARTED); + service.setName("jenkins"); + Artifact artifact = new Artifact(); + artifact.setType(TypeEnum.DOCKER); + artifact.setId("jenkins:latest"); + Resource resource = new Resource(); + resource.setCpus(1); + resource.setMemory("2048"); + List components = new ArrayList(); + Component c = new Component(); + c.setName("jenkins"); + c.setNumberOfContainers(2L); + c.setArtifact(artifact); + c.setLaunchCommand(""); + c.setResource(resource); + components.add(c); + service.setComponents(components); + final Response actual = apiServer.updateService("jenkins", + service); + assertEquals("start service is ", actual.getStatus(), + Response.status(Status.OK).build().getStatus()); + } + + @Test + public void testBadStopServices() { + Service service = new Service(); + service.setState(ServiceState.STOPPED); + service.setName("no-jenkins"); + Artifact artifact = new Artifact(); + artifact.setType(TypeEnum.DOCKER); + artifact.setId("jenkins:latest"); + Resource resource = new Resource(); + resource.setCpus(1); + resource.setMemory("2048"); + List components = new ArrayList(); + Component c = new Component(); + c.setName("no-jenkins"); + c.setNumberOfContainers(-1L); + c.setArtifact(artifact); + c.setLaunchCommand(""); + c.setResource(resource); + components.add(c); + service.setComponents(components); + System.out.println("before stop"); + final Response actual = apiServer.updateService("no-jenkins", + service); + assertEquals("stop service is ", actual.getStatus(), + Response.status(Status.BAD_REQUEST).build().getStatus()); + } + + @Test + public void testGoodStopServices() { + Service service = new Service(); + service.setState(ServiceState.STARTED); + service.setName("jenkins"); + Artifact artifact = new Artifact(); + artifact.setType(TypeEnum.DOCKER); + artifact.setId("jenkins:latest"); + Resource resource = new Resource(); + resource.setCpus(1); + resource.setMemory("2048"); + List components = new ArrayList(); + Component c = new Component(); + c.setName("jenkins"); + c.setNumberOfContainers(-1L); + c.setArtifact(artifact); + c.setLaunchCommand(""); + c.setResource(resource); + components.add(c); + service.setComponents(components); + System.out.println("before stop"); + final Response actual = apiServer.updateService("jenkins", + service); + assertEquals("stop service is ", actual.getStatus(), + Response.status(Status.OK).build().getStatus()); + } + + @Test + public void testUpdateService() { + Service service = new Service(); + service.setState(ServiceState.STARTED); + service.setName("no-jenkins"); + Artifact artifact = new Artifact(); + artifact.setType(TypeEnum.DOCKER); + artifact.setId("jenkins:latest"); + Resource resource = new Resource(); + resource.setCpus(1); + resource.setMemory("2048"); + List components = new ArrayList(); + Component c = new Component(); + c.setName("no-jenkins"); + c.setNumberOfContainers(-1L); + c.setArtifact(artifact); + c.setLaunchCommand(""); + c.setResource(resource); + components.add(c); + service.setComponents(components); + System.out.println("before stop"); + final Response actual = apiServer.updateService("no-jenkins", + service); + assertEquals("update service is ", actual.getStatus(), + Response.status(Status.INTERNAL_SERVER_ERROR) + .build().getStatus()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml index 716fdb7..fd85283 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml @@ -34,5 +34,6 @@ hadoop-yarn-services-core + hadoop-yarn-services-api diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml index 4fb579c..d8a9139 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml @@ -37,7 +37,6 @@ hadoop-yarn-applications-distributedshell hadoop-yarn-applications-unmanaged-am-launcher hadoop-yarn-services - hadoop-yarn-services-api