diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CompoundPlacementConstraint.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CompoundPlacementConstraint.java new file mode 100644 index 0000000..77b28d3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CompoundPlacementConstraint.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * {@code CompoundPlacementConstraint} represents a constraint expression of the + * form AND, OR or DELAYED_OR that consists of other + * {@link PlacementConstraint}s. + */ +@Public +@Unstable +public abstract class CompoundPlacementConstraint { + + @Public + @Unstable + public enum CompoundType { + AND, OR, DELAYED_OR + } + + @Public + @Unstable + public enum DelayUnit { + MILLISECONDS, HEARTBEATS + } + + @Public + @Unstable + public static CompoundPlacementConstraint newInstance( + CompoundType compoundType, List childConstraints) { + return CompoundPlacementConstraint.newInstance(compoundType, + childConstraints, null, null); + } + + @Public + @Unstable + public static CompoundPlacementConstraint newInstance( + CompoundType compoundType, List childConstraints, + List schedulingDelays, DelayUnit delayUnit) { + CompoundPlacementConstraint constraint = + Records.newRecord(CompoundPlacementConstraint.class); + constraint.setCompoundType(compoundType); + constraint.setChildConstraints(childConstraints); + constraint.setSchedulingDelays(schedulingDelays); + constraint.setDelayUnit(delayUnit); + return constraint; + } + + public abstract CompoundType getCompoundType(); + + public abstract void setCompoundType(CompoundType compoundType); + + public abstract List getChildConstraints(); + + public abstract void setChildConstraints( + List childConstraints); + + public abstract List getSchedulingDelays(); + + public abstract void setSchedulingDelays(List schedulingDelays); + + public abstract DelayUnit getDelayUnit(); + + public abstract void setDelayUnit(DelayUnit delayUnit); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PlacementConstraint.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PlacementConstraint.java new file mode 100644 index 0000000..64df06e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PlacementConstraint.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * {@code PlacementConstraint} represents a placement constraint for a resource + * allocation. Placement constraints can be either simple ones + * ({@link SimplePlacementConstraint}) such as affinity to another allocation, + * or more complex expressions ({@link CompoundPlacementConstraint}) such as + * AND/OR expressions of other constraints. + */ +@Public +@Unstable +public abstract class PlacementConstraint { + @Public + @Unstable + public static PlacementConstraint newInstance( + SimplePlacementConstraint simpleConstraint) { + PlacementConstraint constraint = + Records.newRecord(PlacementConstraint.class); + constraint.setSimpleConstraint(simpleConstraint); + return constraint; + } + + @Public + @Unstable + public static PlacementConstraint newInstance( + CompoundPlacementConstraint compoundConstraint) { + PlacementConstraint constraint = + Records.newRecord(PlacementConstraint.class); + constraint.setCompoundConstraint(compoundConstraint); + return constraint; + } + + @Public + @Unstable + public abstract SimplePlacementConstraint getSimpleConstraint(); + + @Public + @Unstable + public abstract void setSimpleConstraint( + SimplePlacementConstraint simpleConstraint); + + @Public + @Unstable + public abstract CompoundPlacementConstraint getCompoundConstraint(); + + @Public + @Unstable + public abstract void setCompoundConstraint( + CompoundPlacementConstraint compoundConstraint); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PlacementConstraintTarget.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PlacementConstraintTarget.java new file mode 100644 index 0000000..14284e2 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PlacementConstraintTarget.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * The {@code PlacementConstraintTarget} is used to define the target + * expressions in a simple target constraint (see + * {@link SimplePlacementConstraint}). + */ +@Public +@Unstable +public abstract class PlacementConstraintTarget { + + @Public + @Unstable + public enum TargetType { + NODE_ATTRIBUTE, ALLOCATION_TAG + } + + @Public + @Unstable + public enum TargetOperator { + IN, NOT_IN + } + + @Public + @Unstable + public static PlacementConstraintTarget newInstance(TargetType targetType, + String targetKey, TargetOperator targetOperator, + Set targetValues) { + PlacementConstraintTarget target = + Records.newRecord(PlacementConstraintTarget.class); + target.setTargetType(targetType); + target.setTargetKey(targetKey); + target.setTargetOperator(targetOperator); + target.setTargetValues(targetValues); + return target; + } + + @Public + @Unstable + public static PlacementConstraintTarget newInstance(TargetType targetType, + TargetOperator targetOperator, Set targetValues) { + // A target on allocation tags does not have a targetKey. + // TODO: Should this be checked here or should it be part of a validation? + return PlacementConstraintTarget.newInstance(targetType, null, + targetOperator, targetValues); + } + + @Public + @Unstable + public abstract TargetType getTargetType(); + + @Public + @Unstable + public abstract void setTargetType(TargetType targetType); + + @Public + @Unstable + public abstract String getTargetKey(); + + @Public + @Unstable + public abstract void setTargetKey(String targetKey); + + @Public + @Unstable + public abstract TargetOperator getTargetOperator(); + + @Public + @Unstable + public abstract void setTargetOperator(TargetOperator targetOperator); + + @Public + @Unstable + public abstract Set getTargetValues(); + + @Public + @Unstable + public abstract void setTargetValues(Set targetValues); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SimplePlacementConstraint.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SimplePlacementConstraint.java new file mode 100644 index 0000000..5d5ba1d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SimplePlacementConstraint.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * {@code SimplePlacementConstraint} represents a simple + * {@link PlacementConstraint}. There are currently two types of simple + * placement constraints: + *
    + *
  • Target constraints
  • + *
  • Cardinality constraints
  • + *
+ */ +@Public +@Unstable +public abstract class SimplePlacementConstraint { + + @Public + @Unstable + public enum ConstraintType { + TARGET_CONSTRAINT, CARDINALITY_CONSTRAINT + } + + @Public + @Unstable + public static SimplePlacementConstraint newInstance(String scope, + Set targetExpressions) { + SimplePlacementConstraint constraint = + Records.newRecord(SimplePlacementConstraint.class); + constraint.setScope(scope); + constraint.setTargetExpressions(targetExpressions); + // TODO: Should the constraint type be input to the constructor? + constraint.setConstraintType(ConstraintType.TARGET_CONSTRAINT); + return constraint; + } + + @Public + @Unstable + public static SimplePlacementConstraint newInstance(String scope, + int minCardinality, int maxCardinality) { + SimplePlacementConstraint constraint = + Records.newRecord(SimplePlacementConstraint.class); + constraint.setScope(scope); + constraint.setMinCardinality(minCardinality); + constraint.setMaxCardinality(maxCardinality); + constraint.setConstraintType(ConstraintType.CARDINALITY_CONSTRAINT); + return constraint; + } + + @Public + @Unstable + public abstract ConstraintType getConstraintType(); + + @Public + @Unstable + public abstract void setConstraintType(ConstraintType constraintType); + + @Public + @Unstable + public abstract String getScope(); + + @Public + @Unstable + public abstract void setScope(String scope); + + @Public + @Unstable + public abstract Set getTargetExpressions(); + + @Public + @Unstable + public abstract void setTargetExpressions( + Set targetExpressions); + + @Public + @Unstable + public abstract int getMinCardinality(); + + @Public + @Unstable + public abstract void setMinCardinality(int minCardinality); + + @Public + @Unstable + public abstract int getMaxCardinality(); + + @Public + @Unstable + public abstract void setMaxCardinality(int maxCardinality); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 81ebd79..d179475 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -495,6 +495,67 @@ enum SignalContainerCommandProto { FORCEFUL_SHUTDOWN = 3; } +//////////////////////////////////////////////////////////////////////// +////// Placement constraints /////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////// + +message PlacementConstraintProto { + optional SimplePlacementConstraintProto simpleConstraint = 1; + optional CompoundPlacementConstraintProto compoundConstraint = 2; +} + +message SimplePlacementConstraintProto { + enum ConstraintType { + TARGET_CONSTRAINT = 1; + CARDINALITY_CONSTRAINT = 2; + } + + required ConstraintType constraintType = 1; + required string scope = 2; + repeated PlacementConstraintTargetProto targetExpressions = 3; + optional int32 minCardinality = 4; + optional int32 maxCardinality = 5; +} + +message PlacementConstraintTargetProto { + enum TargetType { + NODE_ATTRIBUTE = 1; + ALLOCATION_TAG = 2; + } + + enum TargetOperator { + IN = 1; + NOT_IN = 2; + } + + required TargetType targetType = 1; + optional string targetKey = 2; + required TargetOperator targetOperator = 3; + repeated string targetValues = 4; +} + +message CompoundPlacementConstraintProto { + enum CompoundType { + // All children constraints have to be satisfied. + AND = 1; + // One of the children constraints has to be satisfied. + OR = 2; + // Attempt to satisfy the first child constraint for delays[0] units (e.g., + // millisec or heartbeats). If this fails, try to satisfy the second child + // constraint for delays[1] units and so on. + DELAYED_OR = 3; + } + + enum DelayUnit { + MILLISECONDS = 1; + HEARTBEATS = 2; + } + + required CompoundType compoundType = 1; + repeated PlacementConstraintProto childConstraints = 2; + repeated int64 schedulingDelays = 3; + optional DelayUnit delayUnit = 4 [ default = MILLISECONDS ]; +} //////////////////////////////////////////////////////////////////////// ////// From reservation_protocol ///////////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CompoundPlacementConstraintPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CompoundPlacementConstraintPBImpl.java new file mode 100644 index 0000000..55d66ea --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CompoundPlacementConstraintPBImpl.java @@ -0,0 +1,241 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.CompoundPlacementConstraint; +import org.apache.hadoop.yarn.api.records.PlacementConstraint; +import org.apache.hadoop.yarn.proto.YarnProtos.CompoundPlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.CompoundPlacementConstraintProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto; + +@Private +@Unstable +public class CompoundPlacementConstraintPBImpl + extends CompoundPlacementConstraint { + CompoundPlacementConstraintProto proto = + CompoundPlacementConstraintProto.getDefaultInstance(); + CompoundPlacementConstraintProto.Builder builder = null; + boolean viaProto = false; + + List childConstraints = null; + List schedulingDelays = null; + + public CompoundPlacementConstraintPBImpl() { + builder = CompoundPlacementConstraintProto.newBuilder(); + } + + public CompoundPlacementConstraintPBImpl( + CompoundPlacementConstraintProto proto) { + this.proto = proto; + viaProto = true; + } + + public CompoundPlacementConstraintProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToBuilder() { + if (this.childConstraints != null) { + addChildConstraintsToProto(); + } + if (this.schedulingDelays != null) { + builder.clearSchedulingDelays(); + builder.addAllSchedulingDelays(this.schedulingDelays); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = CompoundPlacementConstraintProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public CompoundPlacementConstraint.CompoundType getCompoundType() { + CompoundPlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasCompoundType()) { + return null; + } + return convertFromProtoFormat(p.getCompoundType()); + } + + @Override + public void setCompoundType( + CompoundPlacementConstraint.CompoundType compoundType) { + maybeInitBuilder(); + if (compoundType == null) { + builder.clearCompoundType(); + return; + } + builder.setCompoundType(convertToProtoFormat(compoundType)); + } + + @Override + public List getChildConstraints() { + initChildConstraints(); + return this.childConstraints; + } + + @Override + public void setChildConstraints(List childConstraints) { + if (childConstraints == null) { + builder.clearChildConstraints(); + } + this.childConstraints = childConstraints; + } + + @Override + public List getSchedulingDelays() { + initSchedulingDelays(); + return this.schedulingDelays; + } + + @Override + public void setSchedulingDelays(List schedulingDelays) { + maybeInitBuilder(); + builder.clearSchedulingDelays(); + this.schedulingDelays = schedulingDelays; + } + + @Override + public CompoundPlacementConstraint.DelayUnit getDelayUnit() { + CompoundPlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasDelayUnit()) { + return null; + } + return convertFromProtoFormat(p.getDelayUnit()); + } + + @Override + public void setDelayUnit(CompoundPlacementConstraint.DelayUnit delayUnit) { + maybeInitBuilder(); + if (delayUnit == null) { + builder.clearDelayUnit(); + return; + } + builder.setDelayUnit(convertToProtoFormat(delayUnit)); + } + + private CompoundPlacementConstraint.CompoundType convertFromProtoFormat( + CompoundPlacementConstraintProto.CompoundType t) { + return ProtoUtils.convertFromProtoFormat(t); + } + + private CompoundPlacementConstraintProto.CompoundType convertToProtoFormat( + CompoundPlacementConstraint.CompoundType t) { + return ProtoUtils.convertToProtoFormat(t); + } + + private CompoundPlacementConstraint.DelayUnit convertFromProtoFormat( + CompoundPlacementConstraintProto.DelayUnit u) { + return ProtoUtils.convertFromProtoFormat(u); + } + + private CompoundPlacementConstraintProto.DelayUnit convertToProtoFormat( + CompoundPlacementConstraint.DelayUnit u) { + return ProtoUtils.convertToProtoFormat(u); + } + + private PlacementConstraintPBImpl convertFromProtoFormat( + PlacementConstraintProto c) { + return new PlacementConstraintPBImpl(c); + } + + private PlacementConstraintProto convertToProtoFormat(PlacementConstraint c) { + return ((PlacementConstraintPBImpl) c).getProto(); + } + + private void addChildConstraintsToProto() { + maybeInitBuilder(); + builder.clearChildConstraints(); + if (childConstraints == null) { + return; + } + Iterable iterable = + new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = childConstraints.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public PlacementConstraintProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllChildConstraints(iterable); + } + + private void initChildConstraints() { + if (this.childConstraints != null) { + return; + } + CompoundPlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getChildConstraintsList(); + childConstraints = new ArrayList<>(); + + for (PlacementConstraintProto c : list) { + childConstraints.add(convertFromProtoFormat(c)); + } + } + + private void initSchedulingDelays() { + if (this.schedulingDelays != null) { + return; + } + CompoundPlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + this.schedulingDelays = new ArrayList<>(); + this.schedulingDelays.addAll(p.getSchedulingDelaysList()); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PlacementConstraintPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PlacementConstraintPBImpl.java new file mode 100644 index 0000000..b594faa --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PlacementConstraintPBImpl.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.CompoundPlacementConstraint; +import org.apache.hadoop.yarn.api.records.PlacementConstraint; +import org.apache.hadoop.yarn.api.records.SimplePlacementConstraint; +import org.apache.hadoop.yarn.proto.YarnProtos.CompoundPlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto; + +@Private +@Unstable +public class PlacementConstraintPBImpl extends PlacementConstraint { + PlacementConstraintProto proto = + PlacementConstraintProto.getDefaultInstance(); + PlacementConstraintProto.Builder builder = null; + boolean viaProto = false; + + private SimplePlacementConstraint simpleConstraint = null; + private CompoundPlacementConstraint compoundConstraint = null; + + public PlacementConstraintPBImpl() { + builder = PlacementConstraintProto.newBuilder(); + } + + public PlacementConstraintPBImpl(PlacementConstraintProto proto) { + this.proto = proto; + viaProto = true; + } + + public PlacementConstraintProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToBuilder() { + if (this.simpleConstraint != null) { + builder.setSimpleConstraint(convertToProtoFormat(this.simpleConstraint)); + } + if (this.compoundConstraint != null) { + builder + .setCompoundConstraint(convertToProtoFormat(this.compoundConstraint)); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = PlacementConstraintProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public SimplePlacementConstraint getSimpleConstraint() { + PlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + if (this.simpleConstraint != null) { + return this.simpleConstraint; + } + if (!p.hasSimpleConstraint()) { + return null; + } + this.simpleConstraint = convertFromProtoFormat(p.getSimpleConstraint()); + return this.simpleConstraint; + } + + @Override + public void setSimpleConstraint(SimplePlacementConstraint simpleConstraint) { + maybeInitBuilder(); + if (simpleConstraint == null) { + builder.clearSimpleConstraint(); + } + this.simpleConstraint = simpleConstraint; + } + + @Override + public CompoundPlacementConstraint getCompoundConstraint() { + PlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + if (this.compoundConstraint != null) { + return this.compoundConstraint; + } + if (!p.hasCompoundConstraint()) { + return null; + } + this.compoundConstraint = convertFromProtoFormat(p.getCompoundConstraint()); + return this.compoundConstraint; + } + + @Override + public void setCompoundConstraint( + CompoundPlacementConstraint compoundConstraint) { + maybeInitBuilder(); + if (compoundConstraint == null) { + builder.clearCompoundConstraint(); + } + this.compoundConstraint = compoundConstraint; + } + + private SimplePlacementConstraintPBImpl convertFromProtoFormat( + SimplePlacementConstraintProto s) { + return new SimplePlacementConstraintPBImpl(s); + } + + private SimplePlacementConstraintProto convertToProtoFormat( + SimplePlacementConstraint s) { + return ((SimplePlacementConstraintPBImpl) s).getProto(); + } + + private CompoundPlacementConstraintPBImpl convertFromProtoFormat( + CompoundPlacementConstraintProto s) { + return new CompoundPlacementConstraintPBImpl(s); + } + + private CompoundPlacementConstraintProto convertToProtoFormat( + CompoundPlacementConstraint s) { + return ((CompoundPlacementConstraintPBImpl) s).getProto(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PlacementConstraintTargetPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PlacementConstraintTargetPBImpl.java new file mode 100644 index 0000000..e6d8ad0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PlacementConstraintTargetPBImpl.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + + +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.PlacementConstraintTarget; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProtoOrBuilder; + +@Private +@Unstable +public class PlacementConstraintTargetPBImpl extends PlacementConstraintTarget { + PlacementConstraintTargetProto proto = + PlacementConstraintTargetProto.getDefaultInstance(); + PlacementConstraintTargetProto.Builder builder = null; + boolean viaProto = false; + + Set targetValues = null; + + public PlacementConstraintTargetPBImpl() { + builder = PlacementConstraintTargetProto.newBuilder(); + } + + public PlacementConstraintTargetPBImpl(PlacementConstraintTargetProto proto) { + this.proto = proto; + viaProto = true; + } + + public PlacementConstraintTargetProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToBuilder() { + if (this.targetValues != null) { + builder.clearTargetValues(); + builder.addAllTargetValues(this.targetValues); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = PlacementConstraintTargetProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public PlacementConstraintTarget.TargetType getTargetType() { + PlacementConstraintTargetProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasTargetType()) { + return null; + } + return convertFromProtoFormat(p.getTargetType()); + } + + @Override + public void setTargetType(PlacementConstraintTarget.TargetType targetType) { + maybeInitBuilder(); + if (targetType == null) { + builder.clearTargetType(); + return; + } + builder.setTargetType(convertToProtoFormat(targetType)); + } + + @Override + public String getTargetKey() { + PlacementConstraintTargetProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasTargetKey()) ? p.getTargetKey() : null; + } + + @Override + public void setTargetKey(String targetKey) { + maybeInitBuilder(); + if (targetKey == null) { + builder.clearTargetKey(); + return; + } + builder.setTargetKey(targetKey); + } + + @Override + public PlacementConstraintTarget.TargetOperator getTargetOperator() { + PlacementConstraintTargetProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasTargetOperator()) { + return null; + } + return convertFromProtoFormat(p.getTargetOperator()); + } + + @Override + public void setTargetOperator( + PlacementConstraintTarget.TargetOperator targetOperator) { + maybeInitBuilder(); + if (targetOperator == null) { + builder.clearTargetOperator(); + return; + } + builder.setTargetOperator(convertToProtoFormat(targetOperator)); + } + + @Override + public Set getTargetValues() { + initTargetValues(); + return this.targetValues; + } + + @Override + public void setTargetValues(Set targetValues) { + maybeInitBuilder(); + builder.clearTargetValues(); + this.targetValues = targetValues; + } + + private PlacementConstraintTarget.TargetType convertFromProtoFormat( + PlacementConstraintTargetProto.TargetType t) { + return ProtoUtils.convertFromProtoFormat(t); + } + + private PlacementConstraintTargetProto.TargetType convertToProtoFormat( + PlacementConstraintTarget.TargetType t) { + return ProtoUtils.convertToProtoFormat(t); + } + + private PlacementConstraintTarget.TargetOperator convertFromProtoFormat( + PlacementConstraintTargetProto.TargetOperator o) { + return ProtoUtils.convertFromProtoFormat(o); + } + + private PlacementConstraintTargetProto.TargetOperator convertToProtoFormat( + PlacementConstraintTarget.TargetOperator o) { + return ProtoUtils.convertToProtoFormat(o); + } + + private void initTargetValues() { + if (this.targetValues != null) { + return; + } + PlacementConstraintTargetProtoOrBuilder p = viaProto ? proto : builder; + this.targetValues = new HashSet<>(); + this.targetValues.addAll(p.getTargetValuesList()); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index 926c757..2ebd667 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -27,23 +27,26 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.CompoundPlacementConstraint; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerUpdateType; -import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.PlacementConstraintTarget; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SimplePlacementConstraint; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; @@ -54,7 +57,11 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryPolicyProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerTypeProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto; @@ -65,12 +72,11 @@ import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestInterpreterProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto; +import org.apache.hadoop.yarn.proto.YarnProtos.CompoundPlacementConstraintProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryPolicyProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerTypeProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerUpdateTypeProto; import org.apache.hadoop.yarn.server.api.ContainerType; @@ -433,6 +439,72 @@ public static UpdateContainerErrorPBImpl convertFromProtoFormat( convertToProtoFormat(UpdateContainerError t) { return ((UpdateContainerErrorPBImpl) t).getProto(); } + + /* + * SimplePlacementConstraint.ConstraintType + */ + public static SimplePlacementConstraintProto.ConstraintType convertToProtoFormat( + SimplePlacementConstraint.ConstraintType t) { + return SimplePlacementConstraintProto.ConstraintType.valueOf(t.name()); + } + + public static SimplePlacementConstraint.ConstraintType convertFromProtoFormat( + SimplePlacementConstraintProto.ConstraintType t) { + return SimplePlacementConstraint.ConstraintType.valueOf(t.name()); + } + + /* + * PlacementConstraintTarget.TargetType + */ + public static PlacementConstraintTargetProto.TargetType convertToProtoFormat( + PlacementConstraintTarget.TargetType t) { + return PlacementConstraintTargetProto.TargetType.valueOf(t.name()); + } + + public static PlacementConstraintTarget.TargetType convertFromProtoFormat( + PlacementConstraintTargetProto.TargetType t) { + return PlacementConstraintTarget.TargetType.valueOf(t.name()); + } + + /* + * PlacementConstraintTarget.TargetOperator + */ + public static PlacementConstraintTargetProto.TargetOperator convertToProtoFormat( + PlacementConstraintTarget.TargetOperator o) { + return PlacementConstraintTargetProto.TargetOperator.valueOf(o.name()); + } + + public static PlacementConstraintTarget.TargetOperator convertFromProtoFormat( + PlacementConstraintTargetProto.TargetOperator o) { + return PlacementConstraintTarget.TargetOperator.valueOf(o.name()); + } + + /* + * CompoundPlacementConstraint.CompoundType + */ + public static CompoundPlacementConstraintProto.CompoundType convertToProtoFormat( + CompoundPlacementConstraint.CompoundType t) { + return CompoundPlacementConstraintProto.CompoundType.valueOf(t.name()); + } + + public static CompoundPlacementConstraint.CompoundType convertFromProtoFormat( + CompoundPlacementConstraintProto.CompoundType t) { + return CompoundPlacementConstraint.CompoundType.valueOf(t.name()); + } + + /* + * CompoundPlacementConstraint.DelayUnit + */ + public static CompoundPlacementConstraintProto.DelayUnit convertToProtoFormat( + CompoundPlacementConstraint.DelayUnit u) { + return CompoundPlacementConstraintProto.DelayUnit.valueOf(u.name()); + } + + public static CompoundPlacementConstraint.DelayUnit convertFromProtoFormat( + CompoundPlacementConstraintProto.DelayUnit u) { + return CompoundPlacementConstraint.DelayUnit.valueOf(u.name()); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SimplePlacementConstraintPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SimplePlacementConstraintPBImpl.java new file mode 100644 index 0000000..c059872 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SimplePlacementConstraintPBImpl.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.PlacementConstraintTarget; +import org.apache.hadoop.yarn.api.records.SimplePlacementConstraint; +import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProtoOrBuilder; + +@Private +@Unstable +public class SimplePlacementConstraintPBImpl extends SimplePlacementConstraint { + SimplePlacementConstraintProto proto = + SimplePlacementConstraintProto.getDefaultInstance(); + SimplePlacementConstraintProto.Builder builder = null; + boolean viaProto = false; + + Set targetExpressions = null; + + public SimplePlacementConstraintPBImpl() { + builder = SimplePlacementConstraintProto.newBuilder(); + } + + public SimplePlacementConstraintPBImpl(SimplePlacementConstraintProto proto) { + this.proto = proto; + viaProto = true; + } + + public SimplePlacementConstraintProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToBuilder() { + if (this.targetExpressions != null) { + addTargetExpressionsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = SimplePlacementConstraintProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public SimplePlacementConstraint.ConstraintType getConstraintType() { + SimplePlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasConstraintType()) { + return null; + } + return convertFromProtoFormat(p.getConstraintType()); + } + + @Override + public void setConstraintType( + SimplePlacementConstraint.ConstraintType constraintType) { + maybeInitBuilder(); + if (constraintType == null) { + builder.clearConstraintType(); + return; + } + builder.setConstraintType(convertToProtoFormat(constraintType)); + } + + @Override + public String getScope() { + SimplePlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasScope()) ? p.getScope() : null; + } + + @Override + public void setScope(String scope) { + maybeInitBuilder(); + if (scope == null) { + builder.clearScope(); + return; + } + builder.setScope(scope); + } + + @Override + public Set getTargetExpressions() { + initLocalTargetExpressions(); + return this.targetExpressions; + } + + @Override + public void setTargetExpressions( + Set targetExpressions) { + if (targetExpressions == null) { + builder.clearTargetExpressions(); + } + this.targetExpressions = targetExpressions; + } + + @Override + public int getMinCardinality() { + SimplePlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + return (p.getMinCardinality()); + } + + @Override + public void setMinCardinality(int minCardinality) { + maybeInitBuilder(); + builder.setMinCardinality(minCardinality); + } + + @Override + public int getMaxCardinality() { + SimplePlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + return (p.getMaxCardinality()); + } + + @Override + public void setMaxCardinality(int maxCardinality) { + maybeInitBuilder(); + builder.setMaxCardinality(maxCardinality); + } + + private SimplePlacementConstraint.ConstraintType convertFromProtoFormat( + SimplePlacementConstraintProto.ConstraintType t) { + return ProtoUtils.convertFromProtoFormat(t); + } + + private SimplePlacementConstraintProto.ConstraintType convertToProtoFormat( + SimplePlacementConstraint.ConstraintType t) { + return ProtoUtils.convertToProtoFormat(t); + } + + private PlacementConstraintTargetPBImpl convertFromProtoFormat( + PlacementConstraintTargetProto t) { + return new PlacementConstraintTargetPBImpl(t); + } + + private PlacementConstraintTargetProto convertToProtoFormat( + PlacementConstraintTarget t) { + return ((PlacementConstraintTargetPBImpl) t).getProto(); + } + + private void addTargetExpressionsToProto() { + maybeInitBuilder(); + builder.clearTargetExpressions(); + if (targetExpressions == null) { + return; + } + Iterable iterable = + new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = + targetExpressions.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public PlacementConstraintTargetProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllTargetExpressions(iterable); + } + + private void initLocalTargetExpressions() { + if (this.targetExpressions != null) { + return; + } + SimplePlacementConstraintProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getTargetExpressionsList(); + targetExpressions = new HashSet<>(); + + for (PlacementConstraintTargetProto t : list) { + targetExpressions.add(convertFromProtoFormat(t)); + } + } +}