diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java new file mode 100644 index 0000000..dc84ea0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -0,0 +1,358 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * {@code PlacementConstraint} represents a placement constraint for a resource + * allocation. + */ +@Public +@Unstable +public class PlacementConstraint { + + /** + * This is the constraint expression tree. + */ + AbstractConstraint constraintExpr; + + public PlacementConstraint(AbstractConstraint constraintExpr) { + this.constraintExpr = constraintExpr; + } + + public AbstractConstraint getConstraintExpr() { + return constraintExpr; + } + + /** + * Interface used to enable the elements of the constraint tree to be visited. + */ + public interface Visitable { + /** + * Visitor pattern. + * + * @param visitor visitor to be used + */ + T accept(Visitor visitor); + + } + + /** + * Visitor API for a constraint tree. + */ + public interface Visitor { + T visit(SingleConstraint constraint); + + T visit(TargetExpression target); + + T visit(TargetConstraint constraint); + + T visit(CardinalityConstraint constraint); + + T visit(And constraint); + + T visit(Or constraint); + + T visit(DelayedOr constraint); + + T visit(TimedPlacementConstraint constraint); + } + + public abstract static class AbstractConstraint implements Visitable { + + } + + public static final String NODE_SCOPE = "node"; + public static final String RACK_SCOPE = "rack"; + + public static class SingleConstraint extends AbstractConstraint { + String scope; + int minCardinality; + int maxCardinality; + Set targetExpressions; + + public SingleConstraint(String scope, int minCardinality, + int maxCardinality, Set targetExpressions) { + this.scope = scope; + this.minCardinality = minCardinality; + this.maxCardinality = maxCardinality; + this.targetExpressions = targetExpressions; + } + + public SingleConstraint(String scope, int minC, int maxC, + TargetExpression... targetExpressions) { + this(scope, minC, maxC, new HashSet<>(Arrays.asList(targetExpressions))); + } + + public String getScope() { + return scope; + } + + public int getMinCardinality() { + return minCardinality; + } + + public int getMaxCardinality() { + return maxCardinality; + } + + public Set getTargetExpressions() { + return targetExpressions; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class TargetExpression implements Visitable { + public enum TargetType { + NODE_ATTRIBUTE, ALLOCATION_TAG, SELF + } + + TargetType targetType; + String targetKey; + Set targetValues; + + public TargetExpression(TargetType targetType, String targetKey, + Set targetValues) { + this.targetType = targetType; + this.targetKey = targetKey; + this.targetValues = targetValues; + } + + public TargetExpression(TargetType targetType) { + this(targetType, null, new HashSet<>()); + } + + public TargetExpression(TargetType targetType, String targetKey, + String... targetValues) { + this(targetType, targetKey, new HashSet<>(Arrays.asList(targetValues))); + } + + public TargetType getTargetType() { + return targetType; + } + + public String getTargetKey() { + return targetKey; + } + + public Set getTargetValues() { + return targetValues; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class TargetConstraint extends AbstractConstraint { + enum TargetOperator { + IN, NOT_IN + } + + TargetOperator op; + String scope; + Set targetExpressions; + + public TargetConstraint(TargetOperator op, String scope, + Set targetExpressions) { + this.op = op; + this.scope = scope; + this.targetExpressions = targetExpressions; + } + + public TargetOperator getOp() { + return op; + } + + public String getScope() { + return scope; + } + + public Set getTargetExpressions() { + return targetExpressions; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class CardinalityConstraint extends AbstractConstraint { + String scope; + int minCardinality; + int maxCardinality; + + public CardinalityConstraint(String scope, int minCardinality, + int maxCardinality) { + this.scope = scope; + this.minCardinality = minCardinality; + this.maxCardinality = maxCardinality; + } + + public String getScope() { + return scope; + } + + public int getMinCardinality() { + return minCardinality; + } + + public int getMaxCardinality() { + return maxCardinality; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public abstract static class CompositeConstraint + extends AbstractConstraint { + public abstract List getChildren(); + } + + public static class And extends CompositeConstraint { + List children; + + public And(List children) { + this.children = children; + } + + public And(AbstractConstraint... children) { + this(Arrays.asList(children)); + } + + @Override + public List getChildren() { + return children; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class Or extends CompositeConstraint { + List children; + + public Or(List children) { + this.children = children; + } + + public Or(AbstractConstraint... children) { + this(Arrays.asList(children)); + } + + @Override + public List getChildren() { + return children; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class DelayedOr + extends CompositeConstraint { + List children = new ArrayList<>(); + + public DelayedOr(List children) { + this.children = children; + } + + public DelayedOr(TimedPlacementConstraint... children) { + this(Arrays.asList(children)); + } + + @Override + public List getChildren() { + return children; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } + + public static class TimedPlacementConstraint implements Visitable { + public enum DelayUnit { + MILLISECONDS, OPPORTUNITIES + } + + AbstractConstraint constraint; + long schedulingDelay; + DelayUnit delayUnit; + + public TimedPlacementConstraint(AbstractConstraint constraint, + long schedulingDelay, DelayUnit delayUnit) { + this.constraint = constraint; + this.schedulingDelay = schedulingDelay; + this.delayUnit = delayUnit; + } + + public TimedPlacementConstraint(AbstractConstraint constraint, + long schedulingDelay) { + this(constraint, schedulingDelay, DelayUnit.MILLISECONDS); + } + + public TimedPlacementConstraint(AbstractConstraint constraint) { + this(constraint, Long.MAX_VALUE, DelayUnit.MILLISECONDS); + } + + public AbstractConstraint getConstraint() { + return constraint; + } + + public void setConstraint(AbstractConstraint constraint) { + this.constraint = constraint; + } + + public long getSchedulingDelay() { + return schedulingDelay; + } + + public DelayUnit getDelayUnit() { + return delayUnit; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visit(this); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 81ebd79..df1ba3b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -495,6 +495,61 @@ enum SignalContainerCommandProto { FORCEFUL_SHUTDOWN = 3; } +//////////////////////////////////////////////////////////////////////// +////// Placement constraints /////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////// + +message PlacementConstraintProto { + optional SimplePlacementConstraintProto simpleConstraint = 1; + optional CompositePlacementConstraintProto compositeConstraint = 2; +} + +message SimplePlacementConstraintProto { + required string scope = 1; + repeated PlacementConstraintTargetProto targetExpressions = 2; + optional int32 minCardinality = 3; + optional int32 maxCardinality = 4; +} + +message PlacementConstraintTargetProto { + enum TargetType { + NODE_ATTRIBUTE = 1; + ALLOCATION_TAG = 2; + SELF = 3; + } + + required TargetType targetType = 1; + optional string targetKey = 2; + repeated string targetValues = 3; +} + +message TimedPlacementConstraintProto { + enum DelayUnit { + MILLISECONDS = 1; + OPPORTUNITIES = 2; + } + + required PlacementConstraintProto placementConstraint = 1; + required int64 schedulingDelay = 2; + optional DelayUnit delayUnit = 3 [ default = MILLISECONDS ]; +} + +message CompositePlacementConstraintProto { + enum CompositeType { + // All children constraints have to be satisfied. + AND = 1; + // One of the children constraints has to be satisfied. + OR = 2; + // Attempt to satisfy the first child constraint for delays[0] units (e.g., + // millisec or heartbeats). If this fails, try to satisfy the second child + // constraint for delays[1] units and so on. + DELAYED_OR = 3; + } + + required CompositeType compositeType = 1; + repeated PlacementConstraintProto childConstraints = 2; + repeated TimedPlacementConstraintProto timedChildConstraints = 3; +} //////////////////////////////////////////////////////////////////////// ////// From reservation_protocol ///////////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java new file mode 100644 index 0000000..61eaaa0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.pb; + +import static org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType.AND; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.proto.YarnProtos; +import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto; + +/** + * {@code PlacementConstraintFromProtoConverter} generates an + * {@link PlacementConstraint.AbstractConstraint} given a + * {@link YarnProtos.PlacementConstraintProto}. + */ +@Public +@Unstable +public class PlacementConstraintFromProtoConverter { + + private PlacementConstraintProto constraintProto; + + public PlacementConstraintFromProtoConverter( + PlacementConstraintProto constraintProto) { + this.constraintProto = constraintProto; + } + + public PlacementConstraint convertFromProto(PlacementConstraintProto proto) { + return new PlacementConstraint(convert(proto)); + } + + public AbstractConstraint convert(PlacementConstraintProto proto) { + return proto.hasSimpleConstraint() ? convert(proto.getSimpleConstraint()) + : convert(proto.getCompositeConstraint()); + } + + private SingleConstraint convert(SimplePlacementConstraintProto proto) { + Set targets = new HashSet<>(); + for (PlacementConstraintTargetProto tp : proto.getTargetExpressionsList()) { + targets.add(convert(tp)); + } + + return new SingleConstraint(proto.getScope(), proto.getMinCardinality(), + proto.getMaxCardinality(), targets); + } + + private TargetExpression convert(PlacementConstraintTargetProto proto) { + return new TargetExpression( + ProtoUtils.convertFromProtoFormat(proto.getTargetType()), + proto.getTargetKey(), new HashSet<>(proto.getTargetValuesList())); + } + + private AbstractConstraint convert(CompositePlacementConstraintProto proto) { + switch (proto.getCompositeType()) { + case AND: + case OR: + List children = new ArrayList<>(); + for (PlacementConstraintProto cp : proto.getChildConstraintsList()) { + children.add(convert(cp)); + } + return (proto.getCompositeType() == AND) ? new And(children) + : new Or(children); + case DELAYED_OR: + List tChildren = new ArrayList<>(); + for (TimedPlacementConstraintProto cp : proto + .getTimedChildConstraintsList()) { + tChildren.add(convert(cp)); + } + return new DelayedOr(tChildren); + default: + throw new YarnRuntimeException( + "Encountered unexpected type of composite constraint."); + } + } + + private TimedPlacementConstraint convert( + TimedPlacementConstraintProto proto) { + AbstractConstraint pConstraint = convert(proto.getPlacementConstraint()); + + return new TimedPlacementConstraint(pConstraint, proto.getSchedulingDelay(), + ProtoUtils.convertFromProtoFormat(proto.getDelayUnit())); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java new file mode 100644 index 0000000..fc13c7a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CompositeConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.proto.YarnProtos; +import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto; + +import com.google.protobuf.GeneratedMessage; + +/** + * {@code PlacementConstraintToProtoConverter} generates a + * {@link YarnProtos.PlacementConstraintProto} given a + * {@link PlacementConstraint.AbstractConstraint}. + */ +@Public +@Unstable +public class PlacementConstraintToProtoConverter + implements PlacementConstraint.Visitor { + + private PlacementConstraint constraint; + + public PlacementConstraintToProtoConverter(PlacementConstraint constraint) { + this.constraint = constraint; + } + + public PlacementConstraintProto toProto() { + return (PlacementConstraintProto) constraint.getConstraintExpr() + .accept(this); + } + + @Override + public GeneratedMessage visit(SingleConstraint constraint) { + SimplePlacementConstraintProto.Builder sb = + SimplePlacementConstraintProto.newBuilder(); + + if (constraint.getScope() != null) { + sb.setScope(constraint.getScope()); + } + sb.setMinCardinality(constraint.getMinCardinality()); + sb.setMaxCardinality(constraint.getMaxCardinality()); + if (constraint.getTargetExpressions() != null) { + for (TargetExpression target : constraint.getTargetExpressions()) { + sb.addTargetExpressions( + (PlacementConstraintTargetProto) target.accept(this)); + } + + } + SimplePlacementConstraintProto sProto = sb.build(); + + // Wrap around PlacementConstraintProto object. + PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder(); + pb.setSimpleConstraint(sProto); + return pb.build(); + } + + @Override + public GeneratedMessage visit(TargetExpression target) { + PlacementConstraintTargetProto.Builder tb = + PlacementConstraintTargetProto.newBuilder(); + + tb.setTargetType(ProtoUtils.convertToProtoFormat(target.getTargetType())); + if (target.getTargetKey() != null) { + tb.setTargetKey(target.getTargetKey()); + } + if (target.getTargetValues() != null) { + tb.addAllTargetValues(target.getTargetValues()); + } + return tb.build(); + } + + @Override + public GeneratedMessage visit(TargetConstraint constraint) { + throw new YarnRuntimeException("Unexpected TargetConstraint found."); + } + + @Override + public GeneratedMessage visit(CardinalityConstraint constraint) { + throw new YarnRuntimeException("Unexpected CardinalityConstraint found."); + } + + private GeneratedMessage visitAndOr( + CompositeConstraint composite, CompositeType type) { + CompositePlacementConstraintProto.Builder cb = + CompositePlacementConstraintProto.newBuilder(); + + cb.setCompositeType(type); + + for (AbstractConstraint c : composite.getChildren()) { + cb.addChildConstraints((PlacementConstraintProto) c.accept(this)); + } + CompositePlacementConstraintProto cProto = cb.build(); + + // Wrap around PlacementConstraintProto object. + PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder(); + pb.setCompositeConstraint(cProto); + return pb.build(); + } + + @Override + public GeneratedMessage visit(And constraint) { + return visitAndOr(constraint, CompositeType.AND); + } + + @Override + public GeneratedMessage visit(Or constraint) { + return visitAndOr(constraint, CompositeType.OR); + } + + @Override + public GeneratedMessage visit(DelayedOr constraint) { + CompositePlacementConstraintProto.Builder cb = + CompositePlacementConstraintProto.newBuilder(); + + cb.setCompositeType(CompositeType.DELAYED_OR); + + for (TimedPlacementConstraint c : constraint.getChildren()) { + cb.addTimedChildConstraints( + (TimedPlacementConstraintProto) c.accept(this)); + } + CompositePlacementConstraintProto cProto = cb.build(); + + // Wrap around PlacementConstraintProto object. + PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder(); + pb.setCompositeConstraint(cProto); + return pb.build(); + } + + @Override + public GeneratedMessage visit(TimedPlacementConstraint constraint) { + TimedPlacementConstraintProto.Builder tb = + TimedPlacementConstraintProto.newBuilder(); + + tb.setDelayUnit(ProtoUtils.convertToProtoFormat(constraint.getDelayUnit())); + tb.setSchedulingDelay(constraint.getSchedulingDelay()); + tb.setPlacementConstraint( + (PlacementConstraintProto) constraint.getConstraint().accept(this)); + + return tb.build(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index fa9c430..8583212 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -32,8 +32,8 @@ import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerUpdateType; -import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -48,29 +48,33 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; import org.apache.hadoop.yarn.proto.YarnProtos; import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryPolicyProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerTypeProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto; import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestInterpreterProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryPolicyProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerTypeProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerUpdateTypeProto; import org.apache.hadoop.yarn.server.api.ContainerType; @@ -81,14 +85,15 @@ @Unstable public class ProtoUtils { - /* * ContainerState */ private static String CONTAINER_STATE_PREFIX = "C_"; + public static ContainerStateProto convertToProtoFormat(ContainerState e) { return ContainerStateProto.valueOf(CONTAINER_STATE_PREFIX + e.name()); } + public static ContainerState convertFromProtoFormat(ContainerStateProto e) { return ContainerState.valueOf(e.name().replace(CONTAINER_STATE_PREFIX, "")); } @@ -97,19 +102,22 @@ public static ContainerState convertFromProtoFormat(ContainerStateProto e) { * NodeState */ private static String NODE_STATE_PREFIX = "NS_"; + public static NodeStateProto convertToProtoFormat(NodeState e) { return NodeStateProto.valueOf(NODE_STATE_PREFIX + e.name()); } + public static NodeState convertFromProtoFormat(NodeStateProto e) { return NodeState.valueOf(e.name().replace(NODE_STATE_PREFIX, "")); } - + /* * NodeId */ public static NodeIdProto convertToProtoFormat(NodeId e) { - return ((NodeIdPBImpl)e).getProto(); + return ((NodeIdPBImpl) e).getProto(); } + public static NodeId convertFromProtoFormat(NodeIdProto e) { return new NodeIdPBImpl(e); } @@ -117,10 +125,13 @@ public static NodeId convertFromProtoFormat(NodeIdProto e) { /* * YarnApplicationState */ - public static YarnApplicationStateProto convertToProtoFormat(YarnApplicationState e) { + public static YarnApplicationStateProto convertToProtoFormat( + YarnApplicationState e) { return YarnApplicationStateProto.valueOf(e.name()); } - public static YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto e) { + + public static YarnApplicationState convertFromProtoFormat( + YarnApplicationStateProto e) { return YarnApplicationState.valueOf(e.name()); } @@ -128,37 +139,42 @@ public static YarnApplicationState convertFromProtoFormat(YarnApplicationStatePr * YarnApplicationAttemptState */ private static String YARN_APPLICATION_ATTEMPT_STATE_PREFIX = "APP_ATTEMPT_"; + public static YarnApplicationAttemptStateProto convertToProtoFormat( YarnApplicationAttemptState e) { return YarnApplicationAttemptStateProto .valueOf(YARN_APPLICATION_ATTEMPT_STATE_PREFIX + e.name()); } + public static YarnApplicationAttemptState convertFromProtoFormat( YarnApplicationAttemptStateProto e) { - return YarnApplicationAttemptState.valueOf(e.name().replace( - YARN_APPLICATION_ATTEMPT_STATE_PREFIX, "")); + return YarnApplicationAttemptState + .valueOf(e.name().replace(YARN_APPLICATION_ATTEMPT_STATE_PREFIX, "")); } /* * ApplicationsRequestScope */ - public static YarnServiceProtos.ApplicationsRequestScopeProto - convertToProtoFormat(ApplicationsRequestScope e) { + public static YarnServiceProtos.ApplicationsRequestScopeProto convertToProtoFormat( + ApplicationsRequestScope e) { return YarnServiceProtos.ApplicationsRequestScopeProto.valueOf(e.name()); } - public static ApplicationsRequestScope convertFromProtoFormat - (YarnServiceProtos.ApplicationsRequestScopeProto e) { + + public static ApplicationsRequestScope convertFromProtoFormat( + YarnServiceProtos.ApplicationsRequestScopeProto e) { return ApplicationsRequestScope.valueOf(e.name()); } /* * ApplicationResourceUsageReport */ - public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) { - return ((ApplicationResourceUsageReportPBImpl)e).getProto(); + public static ApplicationResourceUsageReportProto convertToProtoFormat( + ApplicationResourceUsageReport e) { + return ((ApplicationResourceUsageReportPBImpl) e).getProto(); } - public static ApplicationResourceUsageReport convertFromProtoFormat(ApplicationResourceUsageReportProto e) { + public static ApplicationResourceUsageReport convertFromProtoFormat( + ApplicationResourceUsageReportProto e) { return new ApplicationResourceUsageReportPBImpl(e); } @@ -166,39 +182,52 @@ public static ApplicationResourceUsageReport convertFromProtoFormat(ApplicationR * FinalApplicationStatus */ private static String FINAL_APPLICATION_STATUS_PREFIX = "APP_"; - public static FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus e) { - return FinalApplicationStatusProto.valueOf(FINAL_APPLICATION_STATUS_PREFIX + e.name()); + + public static FinalApplicationStatusProto convertToProtoFormat( + FinalApplicationStatus e) { + return FinalApplicationStatusProto + .valueOf(FINAL_APPLICATION_STATUS_PREFIX + e.name()); } - public static FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto e) { - return FinalApplicationStatus.valueOf(e.name().replace(FINAL_APPLICATION_STATUS_PREFIX, "")); + + public static FinalApplicationStatus convertFromProtoFormat( + FinalApplicationStatusProto e) { + return FinalApplicationStatus + .valueOf(e.name().replace(FINAL_APPLICATION_STATUS_PREFIX, "")); } /* * LocalResourceType */ - public static LocalResourceTypeProto convertToProtoFormat(LocalResourceType e) { + public static LocalResourceTypeProto convertToProtoFormat( + LocalResourceType e) { return LocalResourceTypeProto.valueOf(e.name()); } - public static LocalResourceType convertFromProtoFormat(LocalResourceTypeProto e) { + + public static LocalResourceType convertFromProtoFormat( + LocalResourceTypeProto e) { return LocalResourceType.valueOf(e.name()); } /* * LocalResourceVisibility */ - public static LocalResourceVisibilityProto convertToProtoFormat(LocalResourceVisibility e) { + public static LocalResourceVisibilityProto convertToProtoFormat( + LocalResourceVisibility e) { return LocalResourceVisibilityProto.valueOf(e.name()); } - public static LocalResourceVisibility convertFromProtoFormat(LocalResourceVisibilityProto e) { + + public static LocalResourceVisibility convertFromProtoFormat( + LocalResourceVisibilityProto e) { return LocalResourceVisibility.valueOf(e.name()); } - + /* * AMCommand */ public static AMCommandProto convertToProtoFormat(AMCommand e) { return AMCommandProto.valueOf(e.name()); } + public static AMCommand convertFromProtoFormat(AMCommandProto e) { return AMCommand.valueOf(e.name()); } @@ -214,7 +243,7 @@ public static ByteBuffer convertFromProtoFormat(ByteString byteString) { } public static ByteString convertToProtoFormat(ByteBuffer byteBuffer) { -// return ByteString.copyFrom((ByteBuffer)byteBuffer.duplicate().rewind()); + // return ByteString.copyFrom((ByteBuffer)byteBuffer.duplicate().rewind()); int oldPos = byteBuffer.position(); byteBuffer.rewind(); ByteString bs = ByteString.copyFrom(byteBuffer); @@ -226,9 +255,11 @@ public static ByteString convertToProtoFormat(ByteBuffer byteBuffer) { * QueueState */ private static String QUEUE_STATE_PREFIX = "Q_"; + public static QueueStateProto convertToProtoFormat(QueueState e) { return QueueStateProto.valueOf(QUEUE_STATE_PREFIX + e.name()); } + public static QueueState convertFromProtoFormat(QueueStateProto e) { return QueueState.valueOf(e.name().replace(QUEUE_STATE_PREFIX, "")); } @@ -237,14 +268,15 @@ public static QueueState convertFromProtoFormat(QueueStateProto e) { * QueueACL */ private static String QUEUE_ACL_PREFIX = "QACL_"; + public static QueueACLProto convertToProtoFormat(QueueACL e) { return QueueACLProto.valueOf(QUEUE_ACL_PREFIX + e.name()); } + public static QueueACL convertFromProtoFormat(QueueACLProto e) { return QueueACL.valueOf(e.name().replace(QUEUE_ACL_PREFIX, "")); } - /* * ApplicationAccessType */ @@ -252,14 +284,14 @@ public static QueueACL convertFromProtoFormat(QueueACLProto e) { public static ApplicationAccessTypeProto convertToProtoFormat( ApplicationAccessType e) { - return ApplicationAccessTypeProto.valueOf(APP_ACCESS_TYPE_PREFIX - + e.name()); + return ApplicationAccessTypeProto + .valueOf(APP_ACCESS_TYPE_PREFIX + e.name()); } public static ApplicationAccessType convertFromProtoFormat( ApplicationAccessTypeProto e) { - return ApplicationAccessType.valueOf(e.name().replace( - APP_ACCESS_TYPE_PREFIX, "")); + return ApplicationAccessType + .valueOf(e.name().replace(APP_ACCESS_TYPE_PREFIX, "")); } /* @@ -278,7 +310,7 @@ public static ApplicationTimeoutType convertFromProtoFormat( return ApplicationTimeoutType .valueOf(e.name().replace(APP_TIMEOUT_TYPE_PREFIX, "")); } - + /* * Reservation Request interpreter type */ @@ -298,16 +330,17 @@ public static ReservationRequestInterpreter convertFromProtoFormat( private static final String LOG_AGGREGATION_STATUS_PREFIX = "LOG_"; private static final int LOG_AGGREGATION_STATUS_PREFIX_LEN = LOG_AGGREGATION_STATUS_PREFIX.length(); + public static LogAggregationStatusProto convertToProtoFormat( LogAggregationStatus e) { - return LogAggregationStatusProto.valueOf(LOG_AGGREGATION_STATUS_PREFIX - + e.name()); + return LogAggregationStatusProto + .valueOf(LOG_AGGREGATION_STATUS_PREFIX + e.name()); } public static LogAggregationStatus convertFromProtoFormat( LogAggregationStatusProto e) { - return LogAggregationStatus.valueOf(e.name().substring( - LOG_AGGREGATION_STATUS_PREFIX_LEN)); + return LogAggregationStatus + .valueOf(e.name().substring(LOG_AGGREGATION_STATUS_PREFIX_LEN)); } /* @@ -316,6 +349,7 @@ public static LogAggregationStatus convertFromProtoFormat( public static ContainerTypeProto convertToProtoFormat(ContainerType e) { return ContainerTypeProto.valueOf(e.name()); } + public static ContainerType convertFromProtoFormat(ContainerTypeProto e) { return ContainerType.valueOf(e.name()); } @@ -326,6 +360,7 @@ public static ContainerType convertFromProtoFormat(ContainerTypeProto e) { public static ExecutionTypeProto convertToProtoFormat(ExecutionType e) { return ExecutionTypeProto.valueOf(e.name()); } + public static ExecutionType convertFromProtoFormat(ExecutionTypeProto e) { return ExecutionType.valueOf(e.name()); } @@ -337,6 +372,7 @@ public static ContainerUpdateTypeProto convertToProtoFormat( ContainerUpdateType e) { return ContainerUpdateTypeProto.valueOf(e.name()); } + public static ContainerUpdateType convertFromProtoFormat( ContainerUpdateTypeProto e) { return ContainerUpdateType.valueOf(e.name()); @@ -371,7 +407,7 @@ public static ContainerRetryPolicy convertFromProtoFormat( */ public static ExecutionTypeRequestProto convertToProtoFormat( ExecutionTypeRequest e) { - return ((ExecutionTypeRequestPBImpl)e).getProto(); + return ((ExecutionTypeRequestPBImpl) e).getProto(); } public static ExecutionTypeRequest convertFromProtoFormat( @@ -382,9 +418,8 @@ public static ExecutionTypeRequest convertFromProtoFormat( /* * Container */ - public static YarnProtos.ContainerProto convertToProtoFormat( - Container t) { - return ((ContainerPBImpl)t).getProto(); + public static YarnProtos.ContainerProto convertToProtoFormat(Container t) { + return ((ContainerPBImpl) t).getProto(); } public static ContainerPBImpl convertFromProtoFormat( @@ -416,8 +451,8 @@ public static UpdateContainerRequestPBImpl convertFromProtoFormat( return new UpdateContainerRequestPBImpl(p); } - public static YarnServiceProtos.UpdateContainerRequestProto - convertToProtoFormat(UpdateContainerRequest t) { + public static YarnServiceProtos.UpdateContainerRequestProto convertToProtoFormat( + UpdateContainerRequest t) { return ((UpdateContainerRequestPBImpl) t).getProto(); } @@ -429,10 +464,35 @@ public static UpdateContainerErrorPBImpl convertFromProtoFormat( return new UpdateContainerErrorPBImpl(p); } - public static YarnServiceProtos.UpdateContainerErrorProto - convertToProtoFormat(UpdateContainerError t) { + public static YarnServiceProtos.UpdateContainerErrorProto convertToProtoFormat( + UpdateContainerError t) { return ((UpdateContainerErrorPBImpl) t).getProto(); } -} + /* + * PlacementConstraintTarget.TargetType + */ + public static PlacementConstraintTargetProto.TargetType convertToProtoFormat( + TargetExpression.TargetType t) { + return PlacementConstraintTargetProto.TargetType.valueOf(t.name()); + } + + public static TargetExpression.TargetType convertFromProtoFormat( + PlacementConstraintTargetProto.TargetType t) { + return TargetExpression.TargetType.valueOf(t.name()); + } + + /* + * TimedPlacementConstraint.DelayUnit + */ + public static TimedPlacementConstraintProto.DelayUnit convertToProtoFormat( + TimedPlacementConstraint.DelayUnit u) { + return TimedPlacementConstraintProto.DelayUnit.valueOf(u.name()); + } + public static TimedPlacementConstraint.DelayUnit convertFromProtoFormat( + TimedPlacementConstraintProto.DelayUnit u) { + return TimedPlacementConstraint.DelayUnit.valueOf(u.name()); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java new file mode 100644 index 0000000..81186c4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.ListIterator; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CompositeConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint.TargetOperator; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; + +/** + * This class contains inner classes that define transformation on a + * {@link PlacementConstraint} expression. + */ +@Public +@Unstable +public class PlacementConstraintTransformations { + + /** + * The default implementation of the {@link PlacementConstraint.Visitor} that + * performs a traversal of the constraint tree, performing no actions for the + * lead constraints. + */ + public static class DefaultTransformer + implements PlacementConstraint.Visitor { + + private PlacementConstraint constraint; + + public DefaultTransformer(PlacementConstraint constraint) { + this.constraint = constraint; + } + + /** + * This method performs the transformation of the {@link #constraint}. + * + * @return the transformed placement constraint. + */ + public PlacementConstraint transform() { + AbstractConstraint constraintExpr = constraint.getConstraintExpr(); + + // Visit the constraint tree to perform the transformation. + constraintExpr = constraintExpr.accept(this); + + return new PlacementConstraint(constraintExpr); + } + + @Override + public AbstractConstraint visit(SingleConstraint constraint) { + // Do nothing. + return constraint; + } + + @Override + public AbstractConstraint visit(TargetExpression expression) { + // Do nothing. + return null; + } + + @Override + public AbstractConstraint visit(TargetConstraint constraint) { + // Do nothing. + return constraint; + } + + @Override + public AbstractConstraint visit(CardinalityConstraint constraint) { + // Do nothing. + return constraint; + } + + private AbstractConstraint visitAndOr( + CompositeConstraint constraint) { + for (ListIterator iter = + constraint.getChildren().listIterator(); iter.hasNext();) { + AbstractConstraint child = iter.next(); + child = child.accept(this); + iter.set(child); + } + return constraint; + } + + @Override + public AbstractConstraint visit(And constraint) { + return visitAndOr(constraint); + } + + @Override + public AbstractConstraint visit(Or constraint) { + return visitAndOr(constraint); + } + + @Override + public AbstractConstraint visit(DelayedOr constraint) { + constraint.getChildren().forEach( + child -> child.setConstraint(child.getConstraint().accept(this))); + return constraint; + } + + @Override + public AbstractConstraint visit(TimedPlacementConstraint constraint) { + // Do nothing. + return null; + } + } + + /** + * Visits a {@link PlacementConstraint} tree and substitutes each + * {@link TargetConstraint} and {@link CardinalityConstraint} with an + * equivalent {@link SingleConstraint}. + */ + public static class SingleConstraintTransformer extends DefaultTransformer { + + public SingleConstraintTransformer(PlacementConstraint constraint) { + super(constraint); + } + + @Override + public AbstractConstraint visit(TargetConstraint constraint) { + AbstractConstraint newConstraint; + if (constraint.getOp() == TargetOperator.IN) { + newConstraint = new SingleConstraint(constraint.getScope(), 1, + Integer.MAX_VALUE, constraint.getTargetExpressions()); + } else { + newConstraint = new SingleConstraint(constraint.getScope(), 0, 0, + constraint.getTargetExpressions()); + } + return newConstraint; + } + + @Override + public AbstractConstraint visit(CardinalityConstraint constraint) { + return new SingleConstraint(constraint.getScope(), + constraint.getMinCardinality(), constraint.getMaxCardinality(), + new TargetExpression(TargetExpression.TargetType.SELF)); + } + } + + /** + * Visits a {@link PlacementConstraint} tree and, whenever possible, + * substitutes each {@link SingleConstraint} with a {@link TargetConstraint} + * or a {@link CardinalityConstraint}. When such a substitution is not + * possible, we keep the original {@link SingleConstraint}. + */ + public static class SpecializedConstraintTransformer + extends DefaultTransformer { + + public SpecializedConstraintTransformer(PlacementConstraint constraint) { + super(constraint); + } + + @Override + public AbstractConstraint visit(SingleConstraint constraint) { + AbstractConstraint transformedConstraint = constraint; + // Check if it is a cardinality constraint. + if (constraint.getTargetExpressions().size() == 1) { + TargetExpression targetExpr = + constraint.getTargetExpressions().iterator().next(); + if (targetExpr.getTargetType() == TargetExpression.TargetType.SELF) { + transformedConstraint = new CardinalityConstraint( + constraint.getScope(), constraint.getMinCardinality(), + constraint.getMaxCardinality()); + } + } + // Check if it is a target constraint. + if (constraint.getMinCardinality() == 1 + && constraint.getMaxCardinality() == Integer.MAX_VALUE) { + transformedConstraint = new TargetConstraint(TargetOperator.IN, + constraint.getScope(), constraint.getTargetExpressions()); + } else if (constraint.getMinCardinality() == 0 + && constraint.getMaxCardinality() == 0) { + transformedConstraint = new TargetConstraint(TargetOperator.NOT_IN, + constraint.getScope(), constraint.getTargetExpressions()); + } + + return transformedConstraint; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java new file mode 100644 index 0000000..2439c32 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint; + +/** + * This class contains various static methods for the applications to specify + * placement constraints. + */ +@Public +@Unstable +public class PlacementConstraints { + + // Suppresses default constructor, ensuring non-instantiability. + private PlacementConstraints() { + } + + // Creation of simple constraints. + + public static final String NODE = PlacementConstraint.NODE_SCOPE; + public static final String RACK = PlacementConstraint.RACK_SCOPE; + + public static AbstractConstraint targetIn(String scope, + TargetExpression... targetExpressions) { + return new SingleConstraint(scope, 1, Integer.MAX_VALUE, targetExpressions); + } + + public static AbstractConstraint targetNotIn(String scope, + TargetExpression... targetExpressions) { + return new SingleConstraint(scope, 0, 0, targetExpressions); + } + + public static AbstractConstraint cardinality(String scope, int minCardinality, + int maxCardinality) { + return new SingleConstraint(scope, minCardinality, maxCardinality, + PlacementTargets.self()); + } + + public static AbstractConstraint minCardinality(String scope, + int minCardinality) { + return cardinality(scope, minCardinality, Integer.MAX_VALUE); + } + + public static AbstractConstraint maxCardinality(String scope, + int maxCardinality) { + return cardinality(scope, 0, maxCardinality); + } + + public static AbstractConstraint targetCardinality(String scope, + int minCardinality, int maxCardinality, + TargetExpression... targetExpressions) { + return new SingleConstraint(scope, minCardinality, maxCardinality, + targetExpressions); + } + + // Creation of target expressions to be used in simple constraints. + + public static class PlacementTargets { + + public static TargetExpression self() { + return new TargetExpression(TargetType.SELF); + } + + public static TargetExpression nodeAttribute(String attributeKey, + String... attributeValues) { + return new TargetExpression(TargetType.NODE_ATTRIBUTE, attributeKey, + attributeValues); + } + + public static TargetExpression allocationTag(String attributeKey, + String... allocationTags) { + return new TargetExpression(TargetType.ALLOCATION_TAG, null, + allocationTags); + } + } + + // Creation of compound constraints. + + public static And and(AbstractConstraint... children) { + return new And(children); + } + + public static Or or(AbstractConstraint... children) { + return new Or(children); + } + + public static DelayedOr delayedOr(TimedPlacementConstraint... children) { + return new DelayedOr(children); + } + + // Creation of timed constraints to be used in a DELAYED_OR constraint. + + public static TimedPlacementConstraint timedMillisConstraint( + AbstractConstraint constraint, long delay) { + return new TimedPlacementConstraint(constraint, delay, + TimedPlacementConstraint.DelayUnit.MILLISECONDS); + } + + public static TimedPlacementConstraint timedOpportunitiesConstraint( + AbstractConstraint constraint, long delay) { + return new TimedPlacementConstraint(constraint, delay, + TimedPlacementConstraint.DelayUnit.OPPORTUNITIES); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index b62b4ee..bf1850d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -16,6 +16,7 @@ * limitations under the License. */ package org.apache.hadoop.yarn.api; + import java.io.IOException; import org.apache.commons.lang.math.LongRange; @@ -338,8 +339,8 @@ @BeforeClass public static void setup() throws Exception { typeValueCache.put(LongRange.class, new LongRange(1000, 2000)); - typeValueCache.put(URL.class, URL.newInstance( - "http", "localhost", 8080, "file0")); + typeValueCache.put(URL.class, + URL.newInstance("http", "localhost", 8080, "file0")); typeValueCache.put(SerializedException.class, SerializedException.newInstance(new IOException("exception for test"))); generateByNewInstance(ExecutionTypeRequest.class); @@ -381,9 +382,10 @@ public static void setup() throws Exception { generateByNewInstance(UpdatedContainer.class); // genByNewInstance does not apply to QueueInfo, cause // it is recursive(has sub queues) - typeValueCache.put(QueueInfo.class, QueueInfo.newInstance("root", 1.0f, - 1.0f, 0.1f, null, null, QueueState.RUNNING, ImmutableSet.of("x", "y"), - "x && y", null, false)); + typeValueCache.put(QueueInfo.class, + QueueInfo.newInstance("root", 1.0f, 1.0f, 0.1f, null, null, + QueueState.RUNNING, ImmutableSet.of("x", "y"), "x && y", null, + false)); generateByNewInstance(QueueStatistics.class); generateByNewInstance(QueueUserACLInfo.class); generateByNewInstance(YarnClusterMetrics.class); @@ -406,12 +408,14 @@ public static void setup() throws Exception { @Test public void testAllocateRequestPBImpl() throws Exception { - validatePBImplRecord(AllocateRequestPBImpl.class, AllocateRequestProto.class); + validatePBImplRecord(AllocateRequestPBImpl.class, + AllocateRequestProto.class); } @Test public void testAllocateResponsePBImpl() throws Exception { - validatePBImplRecord(AllocateResponsePBImpl.class, AllocateResponseProto.class); + validatePBImplRecord(AllocateResponsePBImpl.class, + AllocateResponseProto.class); } @Test @@ -733,11 +737,11 @@ public void testApplicationResourceUsageReportPBImpl() throws Exception { public void testApplicationSubmissionContextPBImpl() throws Exception { validatePBImplRecord(ApplicationSubmissionContextPBImpl.class, ApplicationSubmissionContextProto.class); - + ApplicationSubmissionContext ctx = ApplicationSubmissionContext.newInstance(null, null, null, null, null, false, false, 0, Resources.none(), null, false, null, null); - + Assert.assertNotNull(ctx.getResource()); } @@ -779,7 +783,8 @@ public void testContainerPBImpl() throws Exception { @Test public void testContainerReportPBImpl() throws Exception { - validatePBImplRecord(ContainerReportPBImpl.class, ContainerReportProto.class); + validatePBImplRecord(ContainerReportPBImpl.class, + ContainerReportProto.class); } @Test @@ -790,7 +795,8 @@ public void testUpdateContainerRequestPBImpl() throws Exception { @Test public void testContainerStatusPBImpl() throws Exception { - validatePBImplRecord(ContainerStatusPBImpl.class, ContainerStatusProto.class); + validatePBImplRecord(ContainerStatusPBImpl.class, + ContainerStatusProto.class); } @Test @@ -875,7 +881,8 @@ public void testResourcePBImpl() throws Exception { @Test public void testResourceRequestPBImpl() throws Exception { - validatePBImplRecord(ResourceRequestPBImpl.class, ResourceRequestProto.class); + validatePBImplRecord(ResourceRequestPBImpl.class, + ResourceRequestProto.class); } @Test @@ -976,7 +983,8 @@ public void testRefreshSuperUserGroupsConfigurationRequestPBImpl() @Test public void testRefreshSuperUserGroupsConfigurationResponsePBImpl() throws Exception { - validatePBImplRecord(RefreshSuperUserGroupsConfigurationResponsePBImpl.class, + validatePBImplRecord( + RefreshSuperUserGroupsConfigurationResponsePBImpl.class, RefreshSuperUserGroupsConfigurationResponseProto.class); } @@ -1043,13 +1051,13 @@ public void testReservationDeleteResponsePBImpl() throws Exception { @Test public void testReservationListRequestPBImpl() throws Exception { validatePBImplRecord(ReservationListRequestPBImpl.class, - ReservationListRequestProto.class); + ReservationListRequestProto.class); } @Test public void testReservationListResponsePBImpl() throws Exception { validatePBImplRecord(ReservationListResponsePBImpl.class, - ReservationListResponseProto.class); + ReservationListResponseProto.class); } @Test @@ -1057,25 +1065,25 @@ public void testAddToClusterNodeLabelsRequestPBImpl() throws Exception { validatePBImplRecord(AddToClusterNodeLabelsRequestPBImpl.class, AddToClusterNodeLabelsRequestProto.class); } - + @Test public void testAddToClusterNodeLabelsResponsePBImpl() throws Exception { validatePBImplRecord(AddToClusterNodeLabelsResponsePBImpl.class, AddToClusterNodeLabelsResponseProto.class); } - + @Test public void testRemoveFromClusterNodeLabelsRequestPBImpl() throws Exception { validatePBImplRecord(RemoveFromClusterNodeLabelsRequestPBImpl.class, RemoveFromClusterNodeLabelsRequestProto.class); } - + @Test public void testRemoveFromClusterNodeLabelsResponsePBImpl() throws Exception { validatePBImplRecord(RemoveFromClusterNodeLabelsResponsePBImpl.class, RemoveFromClusterNodeLabelsResponseProto.class); } - + @Test public void testGetClusterNodeLabelsRequestPBImpl() throws Exception { validatePBImplRecord(GetClusterNodeLabelsRequestPBImpl.class, @@ -1087,7 +1095,7 @@ public void testGetClusterNodeLabelsResponsePBImpl() throws Exception { validatePBImplRecord(GetClusterNodeLabelsResponsePBImpl.class, GetClusterNodeLabelsResponseProto.class); } - + @Test public void testReplaceLabelsOnNodeRequestPBImpl() throws Exception { validatePBImplRecord(ReplaceLabelsOnNodeRequestPBImpl.class, @@ -1099,7 +1107,7 @@ public void testReplaceLabelsOnNodeResponsePBImpl() throws Exception { validatePBImplRecord(ReplaceLabelsOnNodeResponsePBImpl.class, ReplaceLabelsOnNodeResponseProto.class); } - + @Test public void testGetNodeToLabelsRequestPBImpl() throws Exception { validatePBImplRecord(GetNodesToLabelsRequestPBImpl.class, @@ -1123,13 +1131,12 @@ public void testGetLabelsToNodesResponsePBImpl() throws Exception { validatePBImplRecord(GetLabelsToNodesResponsePBImpl.class, GetLabelsToNodesResponseProto.class); } - + @Test public void testNodeLabelAttributesPBImpl() throws Exception { - validatePBImplRecord(NodeLabelPBImpl.class, - NodeLabelProto.class); + validatePBImplRecord(NodeLabelPBImpl.class, NodeLabelProto.class); } - + @Test public void testCheckForDecommissioningNodesRequestPBImpl() throws Exception { validatePBImplRecord(CheckForDecommissioningNodesRequestPBImpl.class, @@ -1137,7 +1144,8 @@ public void testCheckForDecommissioningNodesRequestPBImpl() throws Exception { } @Test - public void testCheckForDecommissioningNodesResponsePBImpl() throws Exception { + public void testCheckForDecommissioningNodesResponsePBImpl() + throws Exception { validatePBImplRecord(CheckForDecommissioningNodesResponsePBImpl.class, CheckForDecommissioningNodesResponseProto.class); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java new file mode 100644 index 0000000..9407802 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.and; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetNotIn; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.nodeAttribute; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.junit.Test; + +/** + * Test class for the various static methods in + * {@link org.apache.hadoop.yarn.api.resource.PlacementConstraints}. + */ +@Public +@Unstable +public class TestPlacementConstraints { + + @Test + public void testNodeAffinityToTag() { + AbstractConstraint constraint = targetIn(NODE, allocationTag("hbase-m")); + } + + @Test + public void testNodeAntiAffinityToAttribute() { + AbstractConstraint constraint = + targetNotIn(NODE, nodeAttribute("java", "1.8")); + } + + @Test + public void testAndConstraint() { + AbstractConstraint constraint = + and(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3), + targetCardinality(RACK, 2, 10, allocationTag("zk"))); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java index c4f37f6..3c7bd01 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java @@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; /** - * Composite key for outstanding scheduler requests for any schedulable entity. + * CompositeConstraint key for outstanding scheduler requests for any schedulable entity. * Currently it includes {@link Priority}. */ public final class SchedulerRequestKey implements diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index e71ce75..cbad064 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -1195,7 +1195,7 @@ public void testWindowsShellScriptBuilderCommand() throws IOException { assertThat(e.getMessage(), CoreMatchers.containsString(expectedMessage)); } - // Composite tests, from parts: less, exact and + + // CompositeConstraint tests, from parts: less, exact and + builder.command(Arrays.asList( org.apache.commons.lang.StringUtils.repeat("A", 1024), org.apache.commons.lang.StringUtils.repeat("A", 1024),