diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTarget.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTarget.java new file mode 100644 index 0000000..3d10eac --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTarget.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +public class AffinityTarget { + final private AffinityTargetType type; + final private String target; + + // Is it affinity or anti-affinity + private boolean affinity; + + public AffinityTarget(AffinityTargetType type, String target) { + affinity = true; + this.type = type; + this.target = target; + } + + public AffinityTargetType getType() { + return type; + } + + public String getTarget() { + return target; + } + + public void setAffinity(boolean affinity) { + this.affinity = affinity; + } + + public boolean getAffinity() { + return affinity; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTargetType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTargetType.java new file mode 100644 index 0000000..6b47e40 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTargetType.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +public enum AffinityTargetType { + /** + * Affinity to another application + */ + APPLICATION, + + /** + * Affinity to given application tag + */ + ALLOCATION_TAG, + + /** + * Affinity to node attribute + */ + NODE_ATTRIBUTE, +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementSetScope.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementSetScope.java new file mode 100644 index 0000000..e0a7891 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementSetScope.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +public enum PlacementSetScope { + NODE, + RACK, +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategy.java new file mode 100644 index 0000000..936b750 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategy.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.PlacementStrategyBuilder; + +public class PlacementStrategy { + private PlacementStrategy[] subStrategies; + private PlacementStrategyOperatorType op; + private PlacementSetScope scope; + private AffinityTarget[] affinityTargets; + private AffinityTarget[] antiAffinityTargets; + + public PlacementStrategy() { + // Set default values + op = PlacementStrategyOperatorType.AND; + scope = PlacementSetScope.NODE; + } + + public PlacementStrategy[] getSubStrategies() { + return subStrategies; + } + + public void setSubStrategies(PlacementStrategy[] subStrategies) { + this.subStrategies = subStrategies; + } + + public PlacementStrategyOperatorType getOp() { + return op; + } + + public void setOp(PlacementStrategyOperatorType op) { + this.op = op; + } + + public PlacementSetScope getScope() { + return scope; + } + + public void setScope(PlacementSetScope scope) { + this.scope = scope; + } + + public AffinityTarget[] getAffinityTargets() { + return affinityTargets; + } + + public void setAffinityTargets( + AffinityTarget[] affinityTargets) { + this.affinityTargets = affinityTargets; + } + + public AffinityTarget[] getAntiAffinityTargets() { + return antiAffinityTargets; + } + + public void setAntiAffinityTargets(AffinityTarget[] antiAffinityTargets) { + this.antiAffinityTargets = antiAffinityTargets; + } + + public static PlacementStrategyBuilder newBuilder() { + return new PlacementStrategyBuilder(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategyOperatorType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategyOperatorType.java new file mode 100644 index 0000000..a33dbf1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategyOperatorType.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +/** + * Operators to connect different placement strategies + */ +public enum PlacementStrategyOperatorType { + AND, // Default + NOT, + OR, + ORDERED_OR +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequest.java new file mode 100644 index 0000000..748d831 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequest.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.UnifiedResourceRequestBuilder; + +import java.util.Set; + +public class PowerfulResourceRequest { + private long allocationRequestId; + + private Set allocationTags; + + private PlacementStrategy placementStrategy; + + /** + * {@link ResourceRequest#getAllocationRequestId()} + */ + public long getAllocationRequestId() { + return allocationRequestId; + } + + public void setAllocationRequestId(long allocationRequestId) { + this.allocationRequestId = allocationRequestId; + } + + public Set getAllocationTags() { + return allocationTags; + } + + public void setAllocationTags(Set allocationTags) { + this.allocationTags = allocationTags; + } + + public PlacementStrategy getPlacementStrategy() { + return placementStrategy; + } + + public void setPlacementStrategy(PlacementStrategy placementStrategy) { + this.placementStrategy = placementStrategy; + } + + public static UnifiedResourceRequestBuilder newBuilder() { + return new UnifiedResourceRequestBuilder(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequestCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequestCollection.java new file mode 100644 index 0000000..a3c6a5e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequestCollection.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.ResourceRequestCollectionBuilder; + +import java.util.Collection; + +public class PowerfulResourceRequestCollection { + private final Collection requests; + + public PowerfulResourceRequestCollection( + Collection requests) { + this.requests = requests; + } + + public static ResourceRequestCollectionBuilder newBuilder() { + return new ResourceRequestCollectionBuilder(); + } + + public Collection getRequests() { + return requests; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/AffinityTargetsBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/AffinityTargetsBuilder.java new file mode 100644 index 0000000..ad71b63 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/AffinityTargetsBuilder.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.AffinityTarget; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.AffinityTargetType; + +import java.util.ArrayList; +import java.util.List; + +public class AffinityTargetsBuilder { + private List targets = new ArrayList<>(); + private boolean affinity = true; + + /** + * Affinity to a specific application Id, + * If application Id = MY_APPLICATION, it means affinity to itself. + * @param appId + * @return + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder toApplication(String appId) { + targets.add( + new AffinityTarget(AffinityTargetType.APPLICATION, appId)); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder toAllocationTags(String... tags) { + for (String tag : tags) { + targets.add( + new AffinityTarget(AffinityTargetType.ALLOCATION_TAG, tag)); + } + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder toNodeAttribute(String nodeAttributeExpr) { + targets.add(new AffinityTarget(AffinityTargetType.NODE_ATTRIBUTE, + nodeAttributeExpr)); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder affinity(boolean affinity) { + this.affinity = affinity; + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTarget[] build() { + // TODO, add code to check sanity such as affinity to multiple node partition + // is not allowed + for (AffinityTarget target : targets) { + target.setAffinity(affinity); + } + return targets.toArray(new AffinityTarget[0]); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/PlacementStrategyBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/PlacementStrategyBuilder.java new file mode 100644 index 0000000..be126cd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/PlacementStrategyBuilder.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PlacementStrategy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PlacementStrategyOperatorType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PlacementSetScope; + +public class PlacementStrategyBuilder { + private PlacementStrategy strategy = new PlacementStrategy(); + + private void subStrategies(PlacementStrategyBuilder... strategies) { + PlacementStrategy[] array = new PlacementStrategy[strategies.length]; + for (int i = 0; i < strategies.length; i++) { + array[i] = strategies[i].build(); + } + strategy.setSubStrategies(array); + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder and(PlacementStrategyBuilder... strategies) { + strategy.setOp(PlacementStrategyOperatorType.AND); + subStrategies(strategies); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder or(PlacementStrategyBuilder... strategies) { + strategy.setOp(PlacementStrategyOperatorType.OR); + subStrategies(strategies); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder not(PlacementStrategyBuilder... strategies) { + strategy.setOp(PlacementStrategyOperatorType.NOT); + subStrategies(strategies); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder scope(PlacementSetScope scope) { + strategy.setScope(scope); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder affinity( + AffinityTargetsBuilder builder) { + strategy.setAffinityTargets(builder.build()); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder antiAffinity( + AffinityTargetsBuilder builder) { + builder.affinity(false); + strategy.setAntiAffinityTargets(builder.build()); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategy build() { + // TODO, add logics to verify if it is legal + return strategy; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/ResourceRequestCollectionBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/ResourceRequestCollectionBuilder.java new file mode 100644 index 0000000..b063fe0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/ResourceRequestCollectionBuilder.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequestCollection; + +import java.util.HashMap; +import java.util.Map; + +public class ResourceRequestCollectionBuilder { + Map requestBuilders; + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public ResourceRequestCollectionBuilder addRequest( + Long allocationRequestId, + UnifiedResourceRequestBuilder requestBuilder) { + requestBuilders.put(allocationRequestId, requestBuilder); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PowerfulResourceRequestCollection build() { + Map requests = new HashMap<>(); + for (Map.Entry entry : requestBuilders + .entrySet()) { + requests.put(entry.getKey(), entry.getValue().build()); + } + + return new PowerfulResourceRequestCollection(requests.values()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/UnifiedResourceRequestBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/UnifiedResourceRequestBuilder.java new file mode 100644 index 0000000..12b523e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/UnifiedResourceRequestBuilder.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder; + +import com.google.common.collect.Sets; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequest; + +public class UnifiedResourceRequestBuilder { + private PowerfulResourceRequest request = new PowerfulResourceRequest(); + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public UnifiedResourceRequestBuilder allocationTags(String... tags) { + request.setAllocationTags(Sets.newHashSet(tags)); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public UnifiedResourceRequestBuilder placement( + PlacementStrategyBuilder placementStrategyBuilder) { + request.setPlacementStrategy(placementStrategyBuilder.build()); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PowerfulResourceRequest build() { + return request; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/example/Examples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/example/Examples.java new file mode 100644 index 0000000..6a235e4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/example/Examples.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.example; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PlacementStrategy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequestCollection; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.AffinityTargetsBuilder; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.ResourceRequestCollectionBuilder; + +public class Examples { + /* + * Application want 10 containers, do not run >1 container on each host + */ + public PowerfulResourceRequestCollection selfAntiAffinity() { + ApplicationId myApplicationId = null; + + ResourceRequestCollectionBuilder builder = + PowerfulResourceRequestCollection.newBuilder(); + builder.addRequest(12345L, PowerfulResourceRequest.newBuilder() + .allocationTags("regionServer", "service").placement( + PlacementStrategy.newBuilder().antiAffinity( + new AffinityTargetsBuilder() + .toApplication("MY_APPLICATION")))); + + return builder.build(); + } + + /* + * Affinity to another app. + * + * One MR application want to affintiy to a Hbase application's regionServers + */ + public PowerfulResourceRequestCollection affinityToAnotherApp() { + ApplicationId hbaseApplicationId = null; + + ResourceRequestCollectionBuilder builder = + PowerfulResourceRequestCollection.newBuilder(); + builder.addRequest(12345L, + PowerfulResourceRequest.newBuilder().allocationTags("MR", "batch") + .placement(PlacementStrategy.newBuilder().affinity( + new AffinityTargetsBuilder() + .toApplication(hbaseApplicationId.toString()) + .toAllocationTags("region-server")))); + + return builder.build(); + } + + /* + * Affinity to both of node attributes and allocation tags + */ + public PowerfulResourceRequestCollection affinityToNodeAttributesAndAllocationTags() { + + ResourceRequestCollectionBuilder builder = + PowerfulResourceRequestCollection.newBuilder(); + builder.addRequest(12345L, + PowerfulResourceRequest.newBuilder().allocationTags("MR", "batch") + .placement(PlacementStrategy.newBuilder().affinity( + new AffinityTargetsBuilder() + .toAllocationTags("region-server") + .toNodeAttribute("SSD && OS=ubuntu14")))); + + return builder.build(); + } +}