diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index e4a245d3dc3..81fdb039ce9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -404,7 +404,7 @@ public int actionFlex(String appName, Map componentCounts) try { Service service = new Service(); service.setName(appName); - service.setState(ServiceState.FLEX); + service.setState(ServiceState.FLEXING); for (Map.Entry entry : componentCounts.entrySet()) { Component component = new Component(); component.setName(entry.getKey()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 14c77f6a2c2..463ee46ef14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -376,7 +376,7 @@ public Response updateService(@Context HttpServletRequest request, updateServiceData.setName(appName); if (updateServiceData.getState() != null - && updateServiceData.getState() == ServiceState.FLEX) { + && updateServiceData.getState() == ServiceState.FLEXING) { return flexService(updateServiceData, ugi); } // For STOP the app should be running. If already stopped then this diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md index 22f941e090f..88616d516e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md @@ -15,7 +15,7 @@ ## Examples ### Create a simple single-component service with most attribute values as defaults -POST URL - http://localhost:9191/ws/v1/services +POST URL - http://localhost:8088/app/v1/services ##### POST Request JSON ```json @@ -27,7 +27,7 @@ POST URL - http://localhost:9191/ws/v1/services [ { "name": "hello", - "number_of_containers": 1, + "number_of_containers": 2, "artifact": { "id": "nginx:latest", "type": "DOCKER" @@ -36,14 +36,14 @@ POST URL - http://localhost:9191/ws/v1/services "resource": { "cpus": 1, "memory": "256" - } + } } ] } ``` ##### GET Response JSON -GET URL - http://localhost:9191/ws/v1/services/hello-world +GET URL - http://localhost:8088/app/v1/services/hello-world Note, lifetime value of -1 means unlimited lifetime. @@ -54,10 +54,12 @@ Note, lifetime value of -1 means unlimited lifetime. "description": "hello world example", "id": "application_1503963985568_0002", "lifetime": -1, + "state": "STABLE", "components": [ { "name": "hello", "dependencies": [], + "state": "STABLE", "resource": { "cpus": 1, "memory": "256" @@ -70,21 +72,21 @@ Note, lifetime value of -1 means unlimited lifetime. "quicklinks": [], "containers": [ { - "id": "container_e03_1503963985568_0002_01_000001", + "id": "container_e03_1503963985568_0002_01_000002", "ip": "10.22.8.143", - "hostname": "myhost.local", + "hostname": "ctr-e03-1503963985568-0002-01-000002.example.site", "state": "READY", "launch_time": 1504051512412, - "bare_host": "10.22.8.143", + "bare_host": "host100.cloud.com", "component_instance_name": "hello-0" }, { - "id": "container_e03_1503963985568_0002_01_000002", - "ip": "10.22.8.143", - "hostname": "myhost.local", + "id": "container_e03_1503963985568_0002_01_000003", + "ip": "10.22.8.144", + "hostname": "ctr-e03-1503963985568-0002-01-000003.example.site", "state": "READY", "launch_time": 1504051536450, - "bare_host": "10.22.8.143", + "bare_host": "host100.cloud.com", "component_instance_name": "hello-1" } ], @@ -103,7 +105,7 @@ Note, lifetime value of -1 means unlimited lifetime. ``` ### Update to modify the lifetime of a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world +PUT URL - http://localhost:8088/app/v1/services/hello-world ##### PUT Request JSON @@ -115,43 +117,59 @@ Note, irrespective of what the current lifetime value is, this update request wi } ``` ### Stop a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world +PUT URL - http://localhost:8088/app/v1/services/hello-world ##### PUT Request JSON ```json { - "state": "STOPPED" + "state": "STOPPED" } ``` ### Start a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world +PUT URL - http://localhost:8088/app/v1/services/hello-world + +##### PUT Request JSON +```json +{ + "state": "STARTED" +} +``` + +### Update to flex up/down the number of containers (instances) of a component of a service +PUT URL - http://localhost:8088/app/v1/services/hello-world/components/hello ##### PUT Request JSON ```json { - "state": "STARTED" + "number_of_containers": 3 } ``` -### Update to flex up/down the no of containers (instances) of a component of a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello +Alternatively, you can specify the entire "components" section instead. +PUT URL - http://localhost:8088/app/v1/services/hello-world ##### PUT Request JSON ```json { - "name": "hello", - "number_of_containers": 3 + "state": "FLEXING", + "components" : + [ + { + "name": "hello", + "number_of_containers": 3 + } + ] } ``` ### Destroy a service -DELETE URL - http://localhost:9191/ws/v1/services/hello-world +DELETE URL - http://localhost:8088/app/v1/services/hello-world *** ### Create a complicated service - HBase -POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 +POST URL - http://localhost:8088:/app/v1/services/hbase-app-1 ##### POST Request JSON @@ -249,3 +267,163 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 } } ``` + +### Create a service requesting GPUs in addition to CPUs and RAM +POST URL - http://localhost:8088/app/v1/services + +##### POST Request JSON +```json +{ + "name": "hello-world", + "version": "1.0.0", + "description": "hello world example with GPUs", + "components" : + [ + { + "name": "hello", + "number_of_containers": 2, + "artifact": { + "id": "nginx:latest", + "type": "DOCKER" + }, + "launch_command": "./start_nginx.sh", + "resource": { + "cpus": 1, + "memory": "256", + "additional" : { + "yarn.io/gpu" : { + "value" : 4, + "unit" : "" + } + } + } + } + ] +} +``` + +### Create a service with a component requesting anti-affinity placement policy +POST URL - http://localhost:8088/app/v1/services + +##### POST Request JSON +```json +{ + "name": "hello-world", + "version": "1.0.0", + "description": "hello world example with anti-affinity", + "components" : + [ + { + "name": "hello", + "number_of_containers": 3, + "artifact": { + "id": "nginx:latest", + "type": "DOCKER" + }, + "launch_command": "./start_nginx.sh", + "resource": { + "cpus": 1, + "memory": "256" + }, + "placement_policy": { + "constraints": [ + { + "type": "ANTI_AFFINITY", + "scope": "NODE", + "target_tags": [ + "hello" + ] + } + ] + } + } + ] +} +``` + +##### GET Response JSON +GET URL - http://localhost:8088/app/v1/services/hello-world + +Note, that the 3 containers will come up on 3 different nodes. If there are less +than 3 NMs running in the cluster, then all 3 container requests will not be +fulfilled and the service will be in non-STABLE state. + +```json +{ + "name": "hello-world", + "version": "1.0.0", + "description": "hello world example with anti-affinity", + "id": "application_1503963985568_0003", + "lifetime": -1, + "state": "STABLE", + "components": [ + { + "name": "hello", + "dependencies": [], + "state": "STABLE", + "resource": { + "cpus": 1, + "memory": "256" + }, + "placement_policy": { + "constraints": [ + { + "type": "ANTI_AFFINITY", + "scope": "NODE", + "node_attributes": {}, + "node_partitions": [], + "target_tags": [ + "hello" + ] + } + ] + }, + "configuration": { + "properties": {}, + "env": {}, + "files": [] + }, + "quicklinks": [], + "containers": [ + { + "id": "container_e03_1503963985568_0003_01_000002", + "ip": "10.22.8.143", + "hostname": "ctr-e03-1503963985568-0003-01-000002.example.site", + "state": "READY", + "launch_time": 1504051512412, + "bare_host": "host100.cloud.com", + "component_instance_name": "hello-0" + }, + { + "id": "container_e03_1503963985568_0003_01_000003", + "ip": "10.22.8.144", + "hostname": "ctr-e03-1503963985568-0003-01-000003.example.site", + "state": "READY", + "launch_time": 1504051536450, + "bare_host": "host101.cloud.com", + "component_instance_name": "hello-1" + }, + { + "id": "container_e03_1503963985568_0003_01_000004", + "ip": "10.22.8.145", + "hostname": "ctr-e03-1503963985568-0003-01-000004.example.site", + "state": "READY", + "launch_time": 1504051536450, + "bare_host": "host102.cloud.com", + "component_instance_name": "hello-2" + } + ], + "launch_command": "./start_nginx.sh", + "number_of_containers": 1, + "run_privileged_container": false + } + ], + "configuration": { + "properties": {}, + "env": {}, + "files": [] + }, + "quicklinks": {} +} +``` + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index 45b1bc725b1..e16bc9e91d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -36,8 +36,8 @@ info: name: Apache 2.0 url: http://www.apache.org/licenses/LICENSE-2.0.html # the domain of the service -host: host.mycompany.com -port: 9191(default) +host: localhost +port: 8088(default) # array of all schemes that your API supports schemes: - http @@ -249,7 +249,7 @@ definitions: type: string description: The YARN queue that this service should be submitted to. kerberos_principal: - description: The Kerberos Principal of the service + description: The principal info of the user who launches the service. $ref: '#/definitions/KerberosPrincipal' ResourceInformation: description: @@ -280,7 +280,7 @@ definitions: type: object additionalProperties: $ref: '#/definitions/ResourceInformation' - description: Map of resource name to ResourceInformation + description: A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below. PlacementPolicy: description: Advanced placement policy of the components of a service. required: @@ -530,7 +530,8 @@ definitions: - STABLE - STOPPED - FAILED - - FLEX + - FLEXING + - UPGRADING ContainerState: description: The current state of the container of a service. properties: @@ -548,8 +549,10 @@ definitions: type: string description: enum of the state of the component enum: + - INIT - FLEXING - STABLE + - UPGRADING ServiceStatus: description: The current status of a submitted service, returned as a response to the GET API. properties: @@ -571,7 +574,4 @@ definitions: description: The principal name of the user who launches the service. keytab: type: string - description: | - The URI of the kerberos keytab. It supports two modes: - URI starts with "hdfs://": A path on hdfs where the keytab is stored. The keytab will be localized by YARN to each host. - URI starts with "file://": A path on the local host where the keytab is stored. It is assumed that the keytabs are pre-installed by admins before AM launches. + description: The URI of the kerberos keytab. It supports two modes, URI starts with "hdfs\://" - A path on hdfs where the keytab is stored. The keytab will be localized by YARN to each host; URI starts with "file\://" - A path on the local host where the keytab is stored. It is assumed that the keytabs are pre-installed by admins before AM launches. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java index 7ac86d4c66f..3d61fb76196 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/BaseResource.java @@ -30,7 +30,7 @@ /** * Resource location for a service, e.g. - * /ws/v1/services/helloworld + * /app/v1/services/helloworld * **/ public String getUri() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java index 286eaa24972..86f954e9366 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java @@ -29,5 +29,5 @@ @ApiModel(description = "The current state of an service.") @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") public enum ServiceState { - ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEX, UPGRADING; + ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEXING, UPGRADING; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index 429c8c1a0dd..0282aa35d21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -241,7 +241,7 @@ The state of the component |Name|Description|Required|Schema|Default| |----|----|----|----|----| -|state|enum of the state of the component|false|enum (FLEXING, STABLE)|| +|state|enum of the state of the component|false|enum (INIT, FLEXING, STABLE, UPGRADING)|| ### ConfigFile @@ -262,7 +262,7 @@ Set of configuration properties that can be injected into the service components |Name|Description|Required|Schema|Default| |----|----|----|----|----| -|properties|A blob of key-value pairs for configuring YARN service AM.|false|object|| +|properties|A blob of key-value pairs for configuring the YARN service AM.|false|object|| |env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object|| |files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array|| @@ -301,7 +301,7 @@ The kerberos principal info of the user who launches the service. |Name|Description|Required|Schema|Default| |----|----|----|----|----| |principal_name|The principal name of the user who launches the service.|false|string|| -|keytab|The URI of the kerberos keytab. It supports two modes, URI starts with "hdfs://": A path on hdfs where the keytab is stored. The keytab will be localized by YARN to each host; URI starts with "file://": A path on the local host where the keytab is stored. It is assumed that the keytabs are pre-installed by admins before AM launches.|false|string|| +|keytab|The URI of the kerberos keytab. It supports two modes, URI starts with "hdfs\://" - A path on hdfs where the keytab is stored. The keytab will be localized by YARN to each host; URI starts with "file\://" - A path on the local host where the keytab is stored. It is assumed that the keytabs are pre-installed by admins before AM launches.|false|string|| ### PlacementConstraint @@ -367,7 +367,7 @@ Resource determines the amount of resources (vcores, memory, network, etc.) usab |profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string|| |cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)|| |memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string|| -|additional|A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below. | false | object || +|additional|A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below.|false|object|| ### ResourceInformation @@ -400,7 +400,8 @@ a service resource has the following attributes. |state|State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state.|false|ServiceState|| |quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object|| |queue|The YARN queue that this service should be submitted to.|false|string|| -|kerberos_principal | The principal info of the user who launches the service|false|KerberosPrincipal|| +|kerberos_principal|The principal info of the user who launches the service.|false|KerberosPrincipal|| + ### ServiceState @@ -408,7 +409,7 @@ The current state of a service. |Name|Description|Required|Schema|Default| |----|----|----|----|----| -|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEX)|| +|state|enum of the state of the service|false|enum (ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEXING, UPGRADING)|| ### ServiceStatus @@ -533,7 +534,7 @@ PUT URL - http://localhost:8088/app/v1/services/hello-world ##### PUT Request JSON ```json { - "state": "STOPPED" + "state": "STOPPED" } ``` @@ -543,17 +544,34 @@ PUT URL - http://localhost:8088/app/v1/services/hello-world ##### PUT Request JSON ```json { - "state": "STARTED" + "state": "STARTED" } ``` -### Update to flex up/down the no of containers (instances) of a component of a service +### Update to flex up/down the number of containers (instances) of a component of a service PUT URL - http://localhost:8088/app/v1/services/hello-world/components/hello ##### PUT Request JSON ```json { - "number_of_containers": 3 + "number_of_containers": 3 +} +``` + +Alternatively, you can specify the entire "components" section instead. + +PUT URL - http://localhost:8088/app/v1/services/hello-world +##### PUT Request JSON +```json +{ + "state": "FLEXING", + "components" : + [ + { + "name": "hello", + "number_of_containers": 3 + } + ] } ``` @@ -820,3 +838,4 @@ fulfilled and the service will be in non-STABLE state. "quicklinks": {} } ``` +