diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 057e94ebd78..047c09ac8b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -275,6 +275,13 @@ public static ResourceInformation newInstance(String name, String units) { Long.MAX_VALUE); } + public static ResourceInformation newInstance(String name, String units, + long value, Map attributes) { + return ResourceInformation + .newInstance(name, units, value, ResourceTypes.COUNTABLE, 0L, + Long.MAX_VALUE, null, attributes); + } + public static ResourceInformation newInstance(String name, String units, ResourceTypes resourceType) { return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java index 103fffb6f75..e466ce7ea57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java @@ -18,10 +18,12 @@ package org.apache.hadoop.yarn.service.api.records; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import java.util.Map; import java.util.Objects; /** @@ -35,11 +37,25 @@ @SerializedName("unit") private String unit = null; + @SerializedName("attributes") + private Map attributes = null; + public ResourceInformation value(Long value) { this.value = value; return this; } + @ApiModelProperty(value = "") + @JsonProperty("attributes") + public Map getAttributes() { + return attributes == null ? ImmutableMap.of() : attributes; + } + + public ResourceInformation attributes(Map attributes) { + this.attributes = attributes; + return this; + } + /** * Integer value of the resource. * @@ -98,6 +114,8 @@ public String toString() { sb.append("class ResourceInformation {\n"); sb.append(" value: ").append(toIndentedString(value)).append("\n"); sb.append(" unit: ").append(toIndentedString(unit)).append("\n"); + sb.append(" attributes: ").append(toIndentedString(attributes)) + .append("\n"); sb.append("}"); return sb.toString(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java index 526bde0ff13..7833b0618ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java @@ -701,7 +701,8 @@ public void requestContainers(long count) { org.apache.hadoop.yarn.api.records.ResourceInformation.newInstance( entry.getKey(), specInfo.getUnit(), - specInfo.getValue()); + specInfo.getValue(), + specInfo.getAttributes()); resource.setResourceInformation(resourceName, ri); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java index 21e93fadcdd..1307b70c03a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java @@ -426,4 +426,51 @@ public void testContainersReleasedWhenPreLaunchFails() am.getComponent("compa").getPendingInstances().size()); am.stop(); } + + @Test + public void testScheduleWithResourceAttributes() throws Exception { + ApplicationId applicationId = ApplicationId.newInstance(123456, 1); + Service exampleApp = new Service(); + exampleApp.setId(applicationId.toString()); + exampleApp.setName("testScheduleWithResourceAttributes"); + exampleApp.setVersion("v1"); + + List resourceTypeInfos = new ArrayList<>( + ResourceUtils.getResourcesTypeInfo()); + // Add 3rd resource type. + resourceTypeInfos.add(ResourceTypeInfo + .newInstance("test-resource", "", ResourceTypes.COUNTABLE)); + // Reinitialize resource types + ResourceUtils.reinitializeResources(resourceTypeInfos); + + Component serviceCompoent = createComponent("compa", 1, "pwd"); + serviceCompoent.getResource().setResourceInformations( + ImmutableMap.of("test-resource", + new ResourceInformation() + .value(1234L) + .unit("Gi") + .attributes(ImmutableMap.of("k1", "v1", "k2", "v2")))); + exampleApp.addComponent(serviceCompoent); + + MockServiceAM am = new MockServiceAM(exampleApp); + am.init(conf); + am.start(); + + ServiceScheduler serviceScheduler = am.context.scheduler; + AMRMClientAsync amrmClientAsync = + serviceScheduler.getAmRMClient(); + + Collection rr = + amrmClientAsync.getMatchingRequests(0); + Assert.assertEquals(1, rr.size()); + + org.apache.hadoop.yarn.api.records.Resource capability = + rr.iterator().next().getCapability(); + Assert.assertEquals(1234L, capability.getResourceValue("test-resource")); + Assert.assertEquals("Gi", + capability.getResourceInformation("test-resource").getUnits()); + Assert.assertEquals(2, capability.getResourceInformation("test-resource") + .getAttributes().size()); + am.stop(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java index 5fdd2ab0c7c..754b589dce0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java @@ -38,6 +38,7 @@ public static final String EXTERNAL_JSON_0 = "external0.json"; public static final String EXTERNAL_JSON_1 = "external1.json"; public static final String EXTERNAL_JSON_2 = "external2.json"; + public static final String EXTERNAL_JSON_3 = "external3.json"; public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java index 73f7fa163af..25c502f6494 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java @@ -20,7 +20,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.ServiceTestUtils; +import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.api.records.ResourceInformation; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.Configuration; @@ -214,4 +216,20 @@ public void testOverrideExternalConfiguration() throws IOException { other = orig.getComponent("other").getConfiguration(); assertEquals(0, other.getProperties().size()); } + + @Test + public void testSetResourceAttributes() throws IOException { + Service orig = ExampleAppJson.loadResource(EXTERNAL_JSON_3); + Component component = orig.getComponent("volume-service"); + Assert.assertNotNull(component); + Map adResource = component + .getResource().getAdditional(); + Assert.assertNotNull(adResource); + Assert.assertEquals(1, adResource.size()); + Map.Entry volume = adResource + .entrySet().iterator().next(); + Assert.assertEquals("yarn.io/csi-volume", volume.getKey()); + Assert.assertEquals(100L, volume.getValue().getValue().longValue()); + Assert.assertEquals(2, volume.getValue().getAttributes().size()); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json new file mode 100644 index 00000000000..74569bd0ba1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json @@ -0,0 +1,26 @@ +{ + "name": "external-3", + "version": "1.0.0", + "lifetime": "3600", + "components": [ + { + "name": "volume-service", + "launch_command": "sleep 3600", + "number_of_containers": 1, + "resource": { + "": 1, + "memory": "512", + "additional": { + "yarn.io/csi-volume": { + "value": 100, + "unit": "Gi", + "attributes" : { + "driver" : "hostpath", + "mountPath" : "/mnt/data" + } + } + } + } + } + ] +} \ No newline at end of file