commit 7c600fce864c943ddb3c27b0ec8ecd04aefcc214 Author: Wangda Tan Date: Tue Oct 25 11:32:22 2016 -0700 YARN-4902 POC (cherry picked from commit 8b2399e388a342cf9393b82ffadc5732620537ba) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/api/unified/DelayToNextCriteria.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/api/unified/DelayToNextCriteria.java new file mode 100644 index 0000000..e990062 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/api/unified/DelayToNextCriteria.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.api.unified; + +/** + * Decide whether or not we need to relax our criteria + */ +public class DelayToNextCriteria { + private long maxDelayInMilliSec; + private long maxMissedOpportunities; + private boolean automatic; + + /** + * Set relax condition to: + * - Longest time wait before relax + * - Maximum missed opportunities + * Relax when any of above conditions stands + * + * @param maxDelayInMilliSec + * @param maxMissedOpportunities + */ + public DelayToNextCriteria(long maxDelayInMilliSec, long maxMissedOpportunities) { + automatic = false; + this.maxDelayInMilliSec = maxDelayInMilliSec; + this.maxMissedOpportunities = maxMissedOpportunities; + } + + /** + * Let system decide when to relax + */ + public DelayToNextCriteria() { + automatic = true; + } + + /** + * Max wait time before relaxing to next criteria + * @return maxDelayInMilliSec + */ + public long getMaxDelayInMilliSec() { + return maxDelayInMilliSec; + } + + /** + * Max missed opportunities before relaxing to next criteria + * @return maxMissedOpportunities + */ + public long getMaxMissedOpportunities() { + return maxMissedOpportunities; + } + + /** + * If we just want system to decide when to relax + * @return automatic + */ + public boolean isAutomatic() { + return automatic; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTarget.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTarget.java new file mode 100644 index 0000000..3d10eac --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTarget.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +public class AffinityTarget { + final private AffinityTargetType type; + final private String target; + + // Is it affinity or anti-affinity + private boolean affinity; + + public AffinityTarget(AffinityTargetType type, String target) { + affinity = true; + this.type = type; + this.target = target; + } + + public AffinityTargetType getType() { + return type; + } + + public String getTarget() { + return target; + } + + public void setAffinity(boolean affinity) { + this.affinity = affinity; + } + + public boolean getAffinity() { + return affinity; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTargetType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTargetType.java new file mode 100644 index 0000000..1a10c25 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/AffinityTargetType.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +public enum AffinityTargetType { + /** + * Affinity to another application + */ + APPLICATION, + + /** + * Affinity to given application tag + */ + ALLOCATION_TAG, + + /** + * Affinity to given nodes + */ + NODE, + + /** + * Affinity to given racks + */ + RACK, + + /** + * Affinity to node partition + */ + NODE_PARTITION +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementSetScope.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementSetScope.java new file mode 100644 index 0000000..e0a7891 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementSetScope.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +public enum PlacementSetScope { + NODE, + RACK, +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategy.java new file mode 100644 index 0000000..2142e2f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategy.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.api.unified.DelayToNextCriteria; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.PlacementStrategyBuilder; + +public class PlacementStrategy { + private PlacementStrategy[] subStrategies; + private PlacementStrategyOperatorType op; + private PlacementSetScope scope; + private int maximumAllocations; + private AffinityTarget[] affinityTargets; + private AffinityTarget[] antiAffinityTargets; + + private DelayToNextCriteria[] delayToNextCriterias; + + public PlacementStrategy() { + // Set default values + op = PlacementStrategyOperatorType.AND; + scope = PlacementSetScope.NODE; + } + + public PlacementStrategy[] getSubStrategies() { + return subStrategies; + } + + public void setSubStrategies(PlacementStrategy[] subStrategies) { + this.subStrategies = subStrategies; + } + + public PlacementStrategyOperatorType getOp() { + return op; + } + + public void setOp(PlacementStrategyOperatorType op) { + this.op = op; + } + + public PlacementSetScope getScope() { + return scope; + } + + public void setScope(PlacementSetScope scope) { + this.scope = scope; + } + + public int getMaximumAllocations() { + return maximumAllocations; + } + + public void setMaximumAllocations(int maximumAllocations) { + this.maximumAllocations = maximumAllocations; + } + + public AffinityTarget[] getAffinityTargets() { + return affinityTargets; + } + + public void setAffinityTargets( + AffinityTarget[] affinityTargets) { + this.affinityTargets = affinityTargets; + } + + public AffinityTarget[] getAntiAffinityTargets() { + return antiAffinityTargets; + } + + public void setAntiAffinityTargets(AffinityTarget[] antiAffinityTargets) { + this.antiAffinityTargets = antiAffinityTargets; + } + + public DelayToNextCriteria[] getDelayToNextCriterias() { + return delayToNextCriterias; + } + + public void setDelayToNextCriterias( + DelayToNextCriteria[] delayToNextCriterias) { + this.delayToNextCriterias = delayToNextCriterias; + } + + public static PlacementStrategyBuilder newBuilder() { + return new PlacementStrategyBuilder(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategyOperatorType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategyOperatorType.java new file mode 100644 index 0000000..a33dbf1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PlacementStrategyOperatorType.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +/** + * Operators to connect different placement strategies + */ +public enum PlacementStrategyOperatorType { + AND, // Default + NOT, + OR, + ORDERED_OR +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequest.java new file mode 100644 index 0000000..882a3b7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequest.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.UnifiedResourceRequestBuilder; + +import java.util.Set; + +public class PowerfulResourceRequest { + private Priority priority; + + private long allocationRequestId; + + private Resource resource; + + private int maximumAllocation; + + private Set allocationTags; + + private PlacementStrategy placementStrategy; + + /** + * {@link ResourceRequest#getPriority()} + */ + public Priority getPriority() { + return priority; + } + + public void setPriority(Priority priority) { + this.priority = priority; + } + + /** + * {@link ResourceRequest#getAllocationRequestId()} + */ + public long getAllocationRequestId() { + return allocationRequestId; + } + + public void setAllocationRequestId(long allocationRequestId) { + this.allocationRequestId = allocationRequestId; + } + + /** + * {@link ResourceRequest#getCapability()} + */ + public Resource getResource() { + return resource; + } + + public void setResource(Resource resource) { + this.resource = resource; + } + + /** + * {@link ResourceRequest#getNumContainers()} + */ + public int getMaximumAllocation() { + return maximumAllocation; + } + + public void setMaximumAllocation(int maximumAllocation) { + this.maximumAllocation = maximumAllocation; + } + + public Set getAllocationTags() { + return allocationTags; + } + + public void setAllocationTags(Set allocationTags) { + this.allocationTags = allocationTags; + } + + public PlacementStrategy getPlacementStrategy() { + return placementStrategy; + } + + public void setPlacementStrategy(PlacementStrategy placementStrategy) { + this.placementStrategy = placementStrategy; + } + + public static UnifiedResourceRequestBuilder newBuilder() { + return new UnifiedResourceRequestBuilder(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequestCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequestCollection.java new file mode 100644 index 0000000..a3c6a5e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/api/PowerfulResourceRequestCollection.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.ResourceRequestCollectionBuilder; + +import java.util.Collection; + +public class PowerfulResourceRequestCollection { + private final Collection requests; + + public PowerfulResourceRequestCollection( + Collection requests) { + this.requests = requests; + } + + public static ResourceRequestCollectionBuilder newBuilder() { + return new ResourceRequestCollectionBuilder(); + } + + public Collection getRequests() { + return requests; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/AffinityTargetsBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/AffinityTargetsBuilder.java new file mode 100644 index 0000000..7ac4582 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/AffinityTargetsBuilder.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.AffinityTarget; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.AffinityTargetType; + +import java.util.ArrayList; +import java.util.List; + +public class AffinityTargetsBuilder { + private List targets = new ArrayList<>(); + private boolean affinity = true; + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder toApplication(ApplicationId appId) { + targets.add( + new AffinityTarget(AffinityTargetType.APPLICATION, appId.toString())); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder toAllocationTags(String... tags) { + for (String tag : tags) { + targets.add( + new AffinityTarget(AffinityTargetType.ALLOCATION_TAG, tag)); + } + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder toHosts(String... hosts) { + for (String host : hosts) { + targets.add(new AffinityTarget(AffinityTargetType.NODE, host)); + } + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder toRacks(String... racks) { + for (String rack : racks) { + targets.add(new AffinityTarget(AffinityTargetType.RACK, rack)); + } + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder toNodePartition(String partition) { + targets.add( + new AffinityTarget(AffinityTargetType.NODE_PARTITION, partition)); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTargetsBuilder affinity(boolean affinity) { + this.affinity = affinity; + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public AffinityTarget[] build() { + // TODO, add code to check sanity such as affinity to multiple node partition + // is not allowed + for (AffinityTarget target : targets) { + target.setAffinity(affinity); + } + return targets.toArray(new AffinityTarget[0]); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/PlacementStrategyBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/PlacementStrategyBuilder.java new file mode 100644 index 0000000..874f40e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/PlacementStrategyBuilder.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PlacementStrategy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PlacementStrategyOperatorType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.api.unified.DelayToNextCriteria; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PlacementSetScope; + +public class PlacementStrategyBuilder { + private PlacementStrategy strategy = new PlacementStrategy(); + + private void subStrategies(PlacementStrategyBuilder... strategies) { + PlacementStrategy[] array = new PlacementStrategy[strategies.length]; + for (int i = 0; i < strategies.length; i++) { + array[i] = strategies[i].build(); + } + strategy.setSubStrategies(array); + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder and(PlacementStrategyBuilder... strategies) { + strategy.setOp(PlacementStrategyOperatorType.AND); + subStrategies(strategies); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder or(PlacementStrategyBuilder... strategies) { + strategy.setOp(PlacementStrategyOperatorType.OR); + subStrategies(strategies); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder not(PlacementStrategyBuilder... strategies) { + strategy.setOp(PlacementStrategyOperatorType.NOT); + subStrategies(strategies); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder orderedOr( + PlacementStrategyBuilder... strategies) { + strategy.setOp(PlacementStrategyOperatorType.ORDERED_OR); + subStrategies(strategies); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder delayCriteras( + DelayToNextCriteria... delayToNextCriterias) { + strategy.setDelayToNextCriterias(delayToNextCriterias); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder scope(PlacementSetScope scope) { + strategy.setScope(scope); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder maximumAllocation(int maximumAllocaiton) { + strategy.setMaximumAllocations(maximumAllocaiton); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder affinity( + AffinityTargetsBuilder builder) { + strategy.setAffinityTargets(builder.build()); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategyBuilder antiAffinity( + AffinityTargetsBuilder builder) { + builder.affinity(false); + strategy.setAntiAffinityTargets(builder.build()); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PlacementStrategy build() { + // TODO, add logics to verify if it is legal + return strategy; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/ResourceRequestCollectionBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/ResourceRequestCollectionBuilder.java new file mode 100644 index 0000000..b063fe0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/ResourceRequestCollectionBuilder.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequestCollection; + +import java.util.HashMap; +import java.util.Map; + +public class ResourceRequestCollectionBuilder { + Map requestBuilders; + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public ResourceRequestCollectionBuilder addRequest( + Long allocationRequestId, + UnifiedResourceRequestBuilder requestBuilder) { + requestBuilders.put(allocationRequestId, requestBuilder); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PowerfulResourceRequestCollection build() { + Map requests = new HashMap<>(); + for (Map.Entry entry : requestBuilders + .entrySet()) { + requests.put(entry.getKey(), entry.getValue().build()); + } + + return new PowerfulResourceRequestCollection(requests.values()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/UnifiedResourceRequestBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/UnifiedResourceRequestBuilder.java new file mode 100644 index 0000000..8dd153b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/builder/UnifiedResourceRequestBuilder.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder; + +import com.google.common.collect.Sets; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequest; + +public class UnifiedResourceRequestBuilder { + private PowerfulResourceRequest request = new PowerfulResourceRequest(); + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public UnifiedResourceRequestBuilder priority(Priority priority) { + request.setPriority(priority); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public UnifiedResourceRequestBuilder priority(int priority) { + request.setPriority(Priority.newInstance(priority)); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public UnifiedResourceRequestBuilder resource(Resource resource) { + request.setResource(resource); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public UnifiedResourceRequestBuilder maximumAllocation(int maximumAllocation) { + request.setMaximumAllocation(maximumAllocation); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public UnifiedResourceRequestBuilder allocationTags(String... tags) { + request.setAllocationTags(Sets.newHashSet(tags)); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public UnifiedResourceRequestBuilder placement( + PlacementStrategyBuilder placementStrategyBuilder) { + request.setPlacementStrategy(placementStrategyBuilder.build()); + return this; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public PowerfulResourceRequest build() { + return request; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/example/Examples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/example/Examples.java new file mode 100644 index 0000000..4e09fc1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/prr/example/Examples.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.example; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PlacementStrategy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.api.PowerfulResourceRequestCollection; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.AffinityTargetsBuilder; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.prr.builder.ResourceRequestCollectionBuilder; + +public class Examples { + /* + * Application want 10 containers, do not run >1 container on each host + */ + public PowerfulResourceRequestCollection selfAntiAffinity() { + ApplicationId myApplicationId = null; + + ResourceRequestCollectionBuilder builder = + PowerfulResourceRequestCollection.newBuilder(); + builder.addRequest(12345L, PowerfulResourceRequest.newBuilder().allocationTags( + "regionServer", "service") + .maximumAllocation(10) + .priority(3) + .resource(Resource.newInstance(1024, 1)) + .placement( + PlacementStrategy.newBuilder() + .antiAffinity(new AffinityTargetsBuilder() + .toApplication(myApplicationId)))); + + return builder.build(); + } + + /* + * Affinity to another app. + * + * One MR application want to affintiy to a Hbase application's regionServers + */ + public PowerfulResourceRequestCollection affinityToAnotherApp() { + ApplicationId hbaseApplicationId = null; + + ResourceRequestCollectionBuilder builder = + PowerfulResourceRequestCollection.newBuilder(); + builder.addRequest(12345L, PowerfulResourceRequest.newBuilder().allocationTags( + "MR", "batch") + .maximumAllocation(10) + .priority(3) + .resource(Resource.newInstance(1024, 1)) + .placement( + PlacementStrategy.newBuilder() + .affinity(new AffinityTargetsBuilder() + .toApplication(hbaseApplicationId) + .toAllocationTags("region-server")))); + + return builder.build(); + } + + /* + * Application want to run on rack1 or rack2 only, but don't want to run + * host1/host2 from /rack1, and don't want to run host3/host4 on /rack2 + */ + public PowerfulResourceRequestCollection andOrNot() { + ResourceRequestCollectionBuilder builder = + PowerfulResourceRequestCollection.newBuilder(); + builder.addRequest(12345L, PowerfulResourceRequest.newBuilder().allocationTags( + "regionServer", "service") + .maximumAllocation(10) + .priority(3) + .resource(Resource.newInstance(1024, 1)) + .placement( + PlacementStrategy.newBuilder().or( + PlacementStrategy.newBuilder() + .affinity(new AffinityTargetsBuilder().toRacks("/rack1")) + .antiAffinity( + new AffinityTargetsBuilder().toHosts("host1", "host2")), + + PlacementStrategy.newBuilder() + .affinity(new AffinityTargetsBuilder().toRacks("/rack2")) + .antiAffinity(new AffinityTargetsBuilder() + .toHosts("host3", "host4"))))); + + return builder.build(); + } + + /* + * Prefer to run on GPU partition, but also accept CPU partition after try + */ + public PowerfulResourceRequestCollection orderedOr() { + ResourceRequestCollectionBuilder builder = + PowerfulResourceRequestCollection.newBuilder(); + builder.addRequest(12345L, PowerfulResourceRequest.newBuilder().allocationTags( + "tensorflow", "service") + .priority(3) + .maximumAllocation(10) + .resource(Resource.newInstance(1024, 1)) + .placement( + PlacementStrategy.newBuilder().orderedOr( + PlacementStrategy.newBuilder() + .affinity(new AffinityTargetsBuilder() + .toNodePartition("GPU_PARTITION")), + + PlacementStrategy.newBuilder() + .affinity(new AffinityTargetsBuilder() + .toNodePartition("*"))) + .delayCriteras(/* 5 mins */))); + + return builder.build(); + } +}