diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java new file mode 100644 index 0000000..dc84ea0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -0,0 +1,358 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * {@code PlacementConstraint} represents a placement constraint for a resource + * allocation. + */ +@Public +@Unstable +public class PlacementConstraint { + + /** + * This is the constraint expression tree. + */ + AbstractConstraint constraintExpr; + + public PlacementConstraint(AbstractConstraint constraintExpr) { + this.constraintExpr = constraintExpr; + } + + public AbstractConstraint getConstraintExpr() { + return constraintExpr; + } + + /** + * Interface used to enable the elements of the constraint tree to be visited. + */ + public interface Visitable { + /** + * Visitor pattern. + * + * @param visitor visitor to be used + */ + T accept(Visitor visitor); + + } + + /** + * Visitor API for a constraint tree. + */ + public interface Visitor { + T visit(SingleConstraint constraint); + + T visit(TargetExpression target); + + T visit(TargetConstraint constraint); + + T visit(CardinalityConstraint constraint); + + T visit(And constraint); + + T visit(Or constraint); + + T visit(DelayedOr constraint); + + T visit(TimedPlacementConstraint constraint); + } + + public abstract static class AbstractConstraint implements Visitable { + + } + + public static final String NODE_SCOPE = "node"; + public static final String RACK_SCOPE = "rack"; + + public static class SingleConstraint extends AbstractConstraint { + String scope; + int minCardinality; + int maxCardinality; + Set targetExpressions; + + public SingleConstraint(String scope, int minCardinality, + int maxCardinality, Set targetExpressions) { + this.scope = scope; + this.minCardinality = minCardinality; + this.maxCardinality = maxCardinality; + this.targetExpressions = targetExpressions; + } + + public SingleConstraint(String scope, int minC, int maxC, + TargetExpression... targetExpressions) { + this(scope, minC, maxC, new HashSet<>(Arrays.asList(targetExpressions))); + } + + public String getScope() { + return scope; + } + + public int getMinCardinality() { + return minCardinality; + } + + public int getMaxCardinality() { + return maxCardinality; + } + + public Set getTargetExpressions() { + return targetExpressions; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class TargetExpression implements Visitable { + public enum TargetType { + NODE_ATTRIBUTE, ALLOCATION_TAG, SELF + } + + TargetType targetType; + String targetKey; + Set targetValues; + + public TargetExpression(TargetType targetType, String targetKey, + Set targetValues) { + this.targetType = targetType; + this.targetKey = targetKey; + this.targetValues = targetValues; + } + + public TargetExpression(TargetType targetType) { + this(targetType, null, new HashSet<>()); + } + + public TargetExpression(TargetType targetType, String targetKey, + String... targetValues) { + this(targetType, targetKey, new HashSet<>(Arrays.asList(targetValues))); + } + + public TargetType getTargetType() { + return targetType; + } + + public String getTargetKey() { + return targetKey; + } + + public Set getTargetValues() { + return targetValues; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class TargetConstraint extends AbstractConstraint { + enum TargetOperator { + IN, NOT_IN + } + + TargetOperator op; + String scope; + Set targetExpressions; + + public TargetConstraint(TargetOperator op, String scope, + Set targetExpressions) { + this.op = op; + this.scope = scope; + this.targetExpressions = targetExpressions; + } + + public TargetOperator getOp() { + return op; + } + + public String getScope() { + return scope; + } + + public Set getTargetExpressions() { + return targetExpressions; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class CardinalityConstraint extends AbstractConstraint { + String scope; + int minCardinality; + int maxCardinality; + + public CardinalityConstraint(String scope, int minCardinality, + int maxCardinality) { + this.scope = scope; + this.minCardinality = minCardinality; + this.maxCardinality = maxCardinality; + } + + public String getScope() { + return scope; + } + + public int getMinCardinality() { + return minCardinality; + } + + public int getMaxCardinality() { + return maxCardinality; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public abstract static class CompositeConstraint + extends AbstractConstraint { + public abstract List getChildren(); + } + + public static class And extends CompositeConstraint { + List children; + + public And(List children) { + this.children = children; + } + + public And(AbstractConstraint... children) { + this(Arrays.asList(children)); + } + + @Override + public List getChildren() { + return children; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class Or extends CompositeConstraint { + List children; + + public Or(List children) { + this.children = children; + } + + public Or(AbstractConstraint... children) { + this(Arrays.asList(children)); + } + + @Override + public List getChildren() { + return children; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class DelayedOr + extends CompositeConstraint { + List children = new ArrayList<>(); + + public DelayedOr(List children) { + this.children = children; + } + + public DelayedOr(TimedPlacementConstraint... children) { + this(Arrays.asList(children)); + } + + @Override + public List getChildren() { + return children; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class TimedPlacementConstraint implements Visitable { + public enum DelayUnit { + MILLISECONDS, OPPORTUNITIES + } + + AbstractConstraint constraint; + long schedulingDelay; + DelayUnit delayUnit; + + public TimedPlacementConstraint(AbstractConstraint constraint, + long schedulingDelay, DelayUnit delayUnit) { + this.constraint = constraint; + this.schedulingDelay = schedulingDelay; + this.delayUnit = delayUnit; + } + + public TimedPlacementConstraint(AbstractConstraint constraint, + long schedulingDelay) { + this(constraint, schedulingDelay, DelayUnit.MILLISECONDS); + } + + public TimedPlacementConstraint(AbstractConstraint constraint) { + this(constraint, Long.MAX_VALUE, DelayUnit.MILLISECONDS); + } + + public AbstractConstraint getConstraint() { + return constraint; + } + + public void setConstraint(AbstractConstraint constraint) { + this.constraint = constraint; + } + + public long getSchedulingDelay() { + return schedulingDelay; + } + + public DelayUnit getDelayUnit() { + return delayUnit; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 81ebd79..df1ba3b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -495,6 +495,61 @@ enum SignalContainerCommandProto { FORCEFUL_SHUTDOWN = 3; } +//////////////////////////////////////////////////////////////////////// +////// Placement constraints /////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////// + +message PlacementConstraintProto { + optional SimplePlacementConstraintProto simpleConstraint = 1; + optional CompositePlacementConstraintProto compositeConstraint = 2; +} + +message SimplePlacementConstraintProto { + required string scope = 1; + repeated PlacementConstraintTargetProto targetExpressions = 2; + optional int32 minCardinality = 3; + optional int32 maxCardinality = 4; +} + +message PlacementConstraintTargetProto { + enum TargetType { + NODE_ATTRIBUTE = 1; + ALLOCATION_TAG = 2; + SELF = 3; + } + + required TargetType targetType = 1; + optional string targetKey = 2; + repeated string targetValues = 3; +} + +message TimedPlacementConstraintProto { + enum DelayUnit { + MILLISECONDS = 1; + OPPORTUNITIES = 2; + } + + required PlacementConstraintProto placementConstraint = 1; + required int64 schedulingDelay = 2; + optional DelayUnit delayUnit = 3 [ default = MILLISECONDS ]; +} + +message CompositePlacementConstraintProto { + enum CompositeType { + // All children constraints have to be satisfied. + AND = 1; + // One of the children constraints has to be satisfied. + OR = 2; + // Attempt to satisfy the first child constraint for delays[0] units (e.g., + // millisec or heartbeats). If this fails, try to satisfy the second child + // constraint for delays[1] units and so on. + DELAYED_OR = 3; + } + + required CompositeType compositeType = 1; + repeated PlacementConstraintProto childConstraints = 2; + repeated TimedPlacementConstraintProto timedChildConstraints = 3; +} //////////////////////////////////////////////////////////////////////// ////// From reservation_protocol ///////////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java new file mode 100644 index 0000000..61eaaa0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.pb; + +import static org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType.AND; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.proto.YarnProtos; +import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto; + +/** + * {@code PlacementConstraintFromProtoConverter} generates an + * {@link PlacementConstraint.AbstractConstraint} given a + * {@link YarnProtos.PlacementConstraintProto}. + */ +@Public +@Unstable +public class PlacementConstraintFromProtoConverter { + + private PlacementConstraintProto constraintProto; + + public PlacementConstraintFromProtoConverter( + PlacementConstraintProto constraintProto) { + this.constraintProto = constraintProto; + } + + public PlacementConstraint convertFromProto(PlacementConstraintProto proto) { + return new PlacementConstraint(convert(proto)); + } + + public AbstractConstraint convert(PlacementConstraintProto proto) { + return proto.hasSimpleConstraint() ? convert(proto.getSimpleConstraint()) + : convert(proto.getCompositeConstraint()); + } + + private SingleConstraint convert(SimplePlacementConstraintProto proto) { + Set targets = new HashSet<>(); + for (PlacementConstraintTargetProto tp : proto.getTargetExpressionsList()) { + targets.add(convert(tp)); + } + + return new SingleConstraint(proto.getScope(), proto.getMinCardinality(), + proto.getMaxCardinality(), targets); + } + + private TargetExpression convert(PlacementConstraintTargetProto proto) { + return new TargetExpression( + ProtoUtils.convertFromProtoFormat(proto.getTargetType()), + proto.getTargetKey(), new HashSet<>(proto.getTargetValuesList())); + } + + private AbstractConstraint convert(CompositePlacementConstraintProto proto) { + switch (proto.getCompositeType()) { + case AND: + case OR: + List children = new ArrayList<>(); + for (PlacementConstraintProto cp : proto.getChildConstraintsList()) { + children.add(convert(cp)); + } + return (proto.getCompositeType() == AND) ? new And(children) + : new Or(children); + case DELAYED_OR: + List tChildren = new ArrayList<>(); + for (TimedPlacementConstraintProto cp : proto + .getTimedChildConstraintsList()) { + tChildren.add(convert(cp)); + } + return new DelayedOr(tChildren); + default: + throw new YarnRuntimeException( + "Encountered unexpected type of composite constraint."); + } + } + + private TimedPlacementConstraint convert( + TimedPlacementConstraintProto proto) { + AbstractConstraint pConstraint = convert(proto.getPlacementConstraint()); + + return new TimedPlacementConstraint(pConstraint, proto.getSchedulingDelay(), + ProtoUtils.convertFromProtoFormat(proto.getDelayUnit())); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java new file mode 100644 index 0000000..fc13c7a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CompositeConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.proto.YarnProtos; +import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto; + +import com.google.protobuf.GeneratedMessage; + +/** + * {@code PlacementConstraintToProtoConverter} generates a + * {@link YarnProtos.PlacementConstraintProto} given a + * {@link PlacementConstraint.AbstractConstraint}. + */ +@Public +@Unstable +public class PlacementConstraintToProtoConverter + implements PlacementConstraint.Visitor { + + private PlacementConstraint constraint; + + public PlacementConstraintToProtoConverter(PlacementConstraint constraint) { + this.constraint = constraint; + } + + public PlacementConstraintProto toProto() { + return (PlacementConstraintProto) constraint.getConstraintExpr() + .accept(this); + } + + @Override + public GeneratedMessage visit(SingleConstraint constraint) { + SimplePlacementConstraintProto.Builder sb = + SimplePlacementConstraintProto.newBuilder(); + + if (constraint.getScope() != null) { + sb.setScope(constraint.getScope()); + } + sb.setMinCardinality(constraint.getMinCardinality()); + sb.setMaxCardinality(constraint.getMaxCardinality()); + if (constraint.getTargetExpressions() != null) { + for (TargetExpression target : constraint.getTargetExpressions()) { + sb.addTargetExpressions( + (PlacementConstraintTargetProto) target.accept(this)); + } + + } + SimplePlacementConstraintProto sProto = sb.build(); + + // Wrap around PlacementConstraintProto object. + PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder(); + pb.setSimpleConstraint(sProto); + return pb.build(); + } + + @Override + public GeneratedMessage visit(TargetExpression target) { + PlacementConstraintTargetProto.Builder tb = + PlacementConstraintTargetProto.newBuilder(); + + tb.setTargetType(ProtoUtils.convertToProtoFormat(target.getTargetType())); + if (target.getTargetKey() != null) { + tb.setTargetKey(target.getTargetKey()); + } + if (target.getTargetValues() != null) { + tb.addAllTargetValues(target.getTargetValues()); + } + return tb.build(); + } + + @Override + public GeneratedMessage visit(TargetConstraint constraint) { + throw new YarnRuntimeException("Unexpected TargetConstraint found."); + } + + @Override + public GeneratedMessage visit(CardinalityConstraint constraint) { + throw new YarnRuntimeException("Unexpected CardinalityConstraint found."); + } + + private GeneratedMessage visitAndOr( + CompositeConstraint composite, CompositeType type) { + CompositePlacementConstraintProto.Builder cb = + CompositePlacementConstraintProto.newBuilder(); + + cb.setCompositeType(type); + + for (AbstractConstraint c : composite.getChildren()) { + cb.addChildConstraints((PlacementConstraintProto) c.accept(this)); + } + CompositePlacementConstraintProto cProto = cb.build(); + + // Wrap around PlacementConstraintProto object. + PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder(); + pb.setCompositeConstraint(cProto); + return pb.build(); + } + + @Override + public GeneratedMessage visit(And constraint) { + return visitAndOr(constraint, CompositeType.AND); + } + + @Override + public GeneratedMessage visit(Or constraint) { + return visitAndOr(constraint, CompositeType.OR); + } + + @Override + public GeneratedMessage visit(DelayedOr constraint) { + CompositePlacementConstraintProto.Builder cb = + CompositePlacementConstraintProto.newBuilder(); + + cb.setCompositeType(CompositeType.DELAYED_OR); + + for (TimedPlacementConstraint c : constraint.getChildren()) { + cb.addTimedChildConstraints( + (TimedPlacementConstraintProto) c.accept(this)); + } + CompositePlacementConstraintProto cProto = cb.build(); + + // Wrap around PlacementConstraintProto object. + PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder(); + pb.setCompositeConstraint(cProto); + return pb.build(); + } + + @Override + public GeneratedMessage visit(TimedPlacementConstraint constraint) { + TimedPlacementConstraintProto.Builder tb = + TimedPlacementConstraintProto.newBuilder(); + + tb.setDelayUnit(ProtoUtils.convertToProtoFormat(constraint.getDelayUnit())); + tb.setSchedulingDelay(constraint.getSchedulingDelay()); + tb.setPlacementConstraint( + (PlacementConstraintProto) constraint.getConstraint().accept(this)); + + return tb.build(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index fa9c430..d16d23b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -48,6 +48,8 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; import org.apache.hadoop.yarn.proto.YarnProtos; import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto; @@ -61,10 +63,12 @@ import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestInterpreterProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryPolicyProto; @@ -433,6 +437,29 @@ public static UpdateContainerErrorPBImpl convertFromProtoFormat( convertToProtoFormat(UpdateContainerError t) { return ((UpdateContainerErrorPBImpl) t).getProto(); } + + public static PlacementConstraintTargetProto.TargetType convertToProtoFormat( + TargetExpression.TargetType t) { + return PlacementConstraintTargetProto.TargetType.valueOf(t.name()); + } + + public static TargetExpression.TargetType convertFromProtoFormat( + PlacementConstraintTargetProto.TargetType t) { + return TargetExpression.TargetType.valueOf(t.name()); + } + + /* + * TimedPlacementConstraint.DelayUnit + */ + public static TimedPlacementConstraintProto.DelayUnit convertToProtoFormat( + TimedPlacementConstraint.DelayUnit u) { + return TimedPlacementConstraintProto.DelayUnit.valueOf(u.name()); + } + + public static TimedPlacementConstraint.DelayUnit convertFromProtoFormat( + TimedPlacementConstraintProto.DelayUnit u) { + return TimedPlacementConstraint.DelayUnit.valueOf(u.name()); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java new file mode 100644 index 0000000..81186c4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.ListIterator; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CompositeConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint.TargetOperator; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; + +/** + * This class contains inner classes that define transformation on a + * {@link PlacementConstraint} expression. + */ +@Public +@Unstable +public class PlacementConstraintTransformations { + + /** + * The default implementation of the {@link PlacementConstraint.Visitor} that + * performs a traversal of the constraint tree, performing no actions for the + * lead constraints. + */ + public static class DefaultTransformer + implements PlacementConstraint.Visitor { + + private PlacementConstraint constraint; + + public DefaultTransformer(PlacementConstraint constraint) { + this.constraint = constraint; + } + + /** + * This method performs the transformation of the {@link #constraint}. + * + * @return the transformed placement constraint. + */ + public PlacementConstraint transform() { + AbstractConstraint constraintExpr = constraint.getConstraintExpr(); + + // Visit the constraint tree to perform the transformation. + constraintExpr = constraintExpr.accept(this); + + return new PlacementConstraint(constraintExpr); + } + + @Override + public AbstractConstraint visit(SingleConstraint constraint) { + // Do nothing. + return constraint; + } + + @Override + public AbstractConstraint visit(TargetExpression expression) { + // Do nothing. + return null; + } + + @Override + public AbstractConstraint visit(TargetConstraint constraint) { + // Do nothing. + return constraint; + } + + @Override + public AbstractConstraint visit(CardinalityConstraint constraint) { + // Do nothing. + return constraint; + } + + private AbstractConstraint visitAndOr( + CompositeConstraint constraint) { + for (ListIterator iter = + constraint.getChildren().listIterator(); iter.hasNext();) { + AbstractConstraint child = iter.next(); + child = child.accept(this); + iter.set(child); + } + return constraint; + } + + @Override + public AbstractConstraint visit(And constraint) { + return visitAndOr(constraint); + } + + @Override + public AbstractConstraint visit(Or constraint) { + return visitAndOr(constraint); + } + + @Override + public AbstractConstraint visit(DelayedOr constraint) { + constraint.getChildren().forEach( + child -> child.setConstraint(child.getConstraint().accept(this))); + return constraint; + } + + @Override + public AbstractConstraint visit(TimedPlacementConstraint constraint) { + // Do nothing. + return null; + } + } + + /** + * Visits a {@link PlacementConstraint} tree and substitutes each + * {@link TargetConstraint} and {@link CardinalityConstraint} with an + * equivalent {@link SingleConstraint}. + */ + public static class SingleConstraintTransformer extends DefaultTransformer { + + public SingleConstraintTransformer(PlacementConstraint constraint) { + super(constraint); + } + + @Override + public AbstractConstraint visit(TargetConstraint constraint) { + AbstractConstraint newConstraint; + if (constraint.getOp() == TargetOperator.IN) { + newConstraint = new SingleConstraint(constraint.getScope(), 1, + Integer.MAX_VALUE, constraint.getTargetExpressions()); + } else { + newConstraint = new SingleConstraint(constraint.getScope(), 0, 0, + constraint.getTargetExpressions()); + } + return newConstraint; + } + + @Override + public AbstractConstraint visit(CardinalityConstraint constraint) { + return new SingleConstraint(constraint.getScope(), + constraint.getMinCardinality(), constraint.getMaxCardinality(), + new TargetExpression(TargetExpression.TargetType.SELF)); + } + } + + /** + * Visits a {@link PlacementConstraint} tree and, whenever possible, + * substitutes each {@link SingleConstraint} with a {@link TargetConstraint} + * or a {@link CardinalityConstraint}. When such a substitution is not + * possible, we keep the original {@link SingleConstraint}. + */ + public static class SpecializedConstraintTransformer + extends DefaultTransformer { + + public SpecializedConstraintTransformer(PlacementConstraint constraint) { + super(constraint); + } + + @Override + public AbstractConstraint visit(SingleConstraint constraint) { + AbstractConstraint transformedConstraint = constraint; + // Check if it is a cardinality constraint. + if (constraint.getTargetExpressions().size() == 1) { + TargetExpression targetExpr = + constraint.getTargetExpressions().iterator().next(); + if (targetExpr.getTargetType() == TargetExpression.TargetType.SELF) { + transformedConstraint = new CardinalityConstraint( + constraint.getScope(), constraint.getMinCardinality(), + constraint.getMaxCardinality()); + } + } + // Check if it is a target constraint. + if (constraint.getMinCardinality() == 1 + && constraint.getMaxCardinality() == Integer.MAX_VALUE) { + transformedConstraint = new TargetConstraint(TargetOperator.IN, + constraint.getScope(), constraint.getTargetExpressions()); + } else if (constraint.getMinCardinality() == 0 + && constraint.getMaxCardinality() == 0) { + transformedConstraint = new TargetConstraint(TargetOperator.NOT_IN, + constraint.getScope(), constraint.getTargetExpressions()); + } + + return transformedConstraint; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java new file mode 100644 index 0000000..2439c32 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; + +/** + * This class contains various static methods for the applications to specify + * placement constraints. + */ +@Public +@Unstable +public class PlacementConstraints { + + // Suppresses default constructor, ensuring non-instantiability. + private PlacementConstraints() { + } + + // Creation of simple constraints. + + public static final String NODE = PlacementConstraint.NODE_SCOPE; + public static final String RACK = PlacementConstraint.RACK_SCOPE; + + public static AbstractConstraint targetIn(String scope, + TargetExpression... targetExpressions) { + return new SingleConstraint(scope, 1, Integer.MAX_VALUE, targetExpressions); + } + + public static AbstractConstraint targetNotIn(String scope, + TargetExpression... targetExpressions) { + return new SingleConstraint(scope, 0, 0, targetExpressions); + } + + public static AbstractConstraint cardinality(String scope, int minCardinality, + int maxCardinality) { + return new SingleConstraint(scope, minCardinality, maxCardinality, + PlacementTargets.self()); + } + + public static AbstractConstraint minCardinality(String scope, + int minCardinality) { + return cardinality(scope, minCardinality, Integer.MAX_VALUE); + } + + public static AbstractConstraint maxCardinality(String scope, + int maxCardinality) { + return cardinality(scope, 0, maxCardinality); + } + + public static AbstractConstraint targetCardinality(String scope, + int minCardinality, int maxCardinality, + TargetExpression... targetExpressions) { + return new SingleConstraint(scope, minCardinality, maxCardinality, + targetExpressions); + } + + // Creation of target expressions to be used in simple constraints. + + public static class PlacementTargets { + + public static TargetExpression self() { + return new TargetExpression(TargetType.SELF); + } + + public static TargetExpression nodeAttribute(String attributeKey, + String... attributeValues) { + return new TargetExpression(TargetType.NODE_ATTRIBUTE, attributeKey, + attributeValues); + } + + public static TargetExpression allocationTag(String attributeKey, + String... allocationTags) { + return new TargetExpression(TargetType.ALLOCATION_TAG, null, + allocationTags); + } + } + + // Creation of compound constraints. + + public static And and(AbstractConstraint... children) { + return new And(children); + } + + public static Or or(AbstractConstraint... children) { + return new Or(children); + } + + public static DelayedOr delayedOr(TimedPlacementConstraint... children) { + return new DelayedOr(children); + } + + // Creation of timed constraints to be used in a DELAYED_OR constraint. + + public static TimedPlacementConstraint timedMillisConstraint( + AbstractConstraint constraint, long delay) { + return new TimedPlacementConstraint(constraint, delay, + TimedPlacementConstraint.DelayUnit.MILLISECONDS); + } + + public static TimedPlacementConstraint timedOpportunitiesConstraint( + AbstractConstraint constraint, long delay) { + return new TimedPlacementConstraint(constraint, delay, + TimedPlacementConstraint.DelayUnit.OPPORTUNITIES); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java new file mode 100644 index 0000000..9407802 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.and; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetNotIn; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.nodeAttribute; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.junit.Test; + +/** + * Test class for the various static methods in + * {@link org.apache.hadoop.yarn.api.resource.PlacementConstraints}. + */ +@Public +@Unstable +public class TestPlacementConstraints { + + @Test + public void testNodeAffinityToTag() { + AbstractConstraint constraint = targetIn(NODE, allocationTag("hbase-m")); + } + + @Test + public void testNodeAntiAffinityToAttribute() { + AbstractConstraint constraint = + targetNotIn(NODE, nodeAttribute("java", "1.8")); + } + + @Test + public void testAndConstraint() { + AbstractConstraint constraint = + and(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3), + targetCardinality(RACK, 2, 10, allocationTag("zk"))); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index e71ce75..cbad064 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -1195,7 +1195,7 @@ public void testWindowsShellScriptBuilderCommand() throws IOException { assertThat(e.getMessage(), CoreMatchers.containsString(expectedMessage)); } - // Composite tests, from parts: less, exact and + + // CompositeConstraint tests, from parts: less, exact and + builder.command(Arrays.asList( org.apache.commons.lang.StringUtils.repeat("A", 1024), org.apache.commons.lang.StringUtils.repeat("A", 1024),