org.bouncycastle
bcprov-jdk16
test
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
new file mode 100644
index 0000000000000000000000000000000000000000..fb65eb2175b423929a54737b8ce0496170e71113
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.Serializable;
+import java.util.Deque;
+
+/**
+ * An {@link Appendable} implementation that considers its {@link #capacity} as
+ * upper bound. While {@link #append(CharSequence) append}ing the lengths of the
+ * past input values are gathered in a {@link Deque deque}.
+ *
+ * When {@link #capacity} would be reached on append, if possible, at head of
+ * {@link #pastInputValues past input values}, some of the {@link String}s are
+ * {@link StringBuilder#delete(int, int) delete}d first considering:
+ *
+ *
+ * - only whole {@link CharSequence}s {@link #append(CharSequence) append}ed
+ * before are deleted
+ * - first-in-first-out, meaning head first semantics
+ * - combined length of all the {@link CharSequence}s deleted in one go should
+ * be at least the length of the new value
+ *
+ *
+ * Note that null values are {@link #append(CharSequence) append}ed
+ * just like in {@link StringBuilder#append(CharSequence) original
+ * implementation}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class BoundedAppender implements Appendable, Serializable, CharSequence {
+ private static final long serialVersionUID = 1L;
+
+ private final int capacity;
+ private final StringBuilder pastInputValues = new StringBuilder();
+ private final Deque pastInputLengths = Lists.newLinkedList();
+
+ /**
+ * Default c'tor, initializing {@link #capacity} to 64 kB.
+ */
+ public BoundedAppender() {
+ this(65_536);
+ }
+
+ /**
+ * Constructor to initialize to any {@link #capacity}.
+ *
+ * @param capacity upper bound of the character count
+ */
+ public BoundedAppender(final int capacity) {
+ Preconditions.checkArgument(capacity > 0, "capacity should be positive");
+
+ this.capacity = capacity;
+ }
+
+ /**
+ * Appends the specified string to this BoundedAppender, considering
+ * {@link #capacity} as upper bound in a way that former {@code append}ed
+ * whole strings are {@link StringBuilder#delete(int, int) delete}d in FIFO
+ * order.
+ *
+ * If {@code csq} is {@code null}, then the four characters {@code "null"} are
+ * appended.
+ *
+ * @param csq the {@link CharSequence} to be appended
+ * @return {@code this}
+ * @throws IllegalArgumentException if {@code csq} doesn't fit into
+ * {@code capacity}
+ * @see Appendable#append(CharSequence)
+ */
+ @Override
+ public BoundedAppender append(CharSequence csq) {
+ csq = ensureNull(csq);
+
+ final int inputLength = csq.length();
+
+ checkAndCut(inputLength);
+ pastInputValues.append(csq);
+ pastInputLengths.add(inputLength);
+
+ return this;
+ }
+
+ private CharSequence ensureNull(final CharSequence csq) {
+ if (csq == null) {
+ return "null";
+ }
+
+ return csq;
+ }
+
+ private boolean shouldCut(final int inputLength) {
+ return lengthToCut(inputLength) > 0;
+ }
+
+ private int lengthToCut(final int inputLength) {
+ return pastInputValues.length() + inputLength - capacity;
+ }
+
+ private void cutAtLeast(final int minimalCutLength) {
+ Preconditions.checkArgument(minimalCutLength > 0, "nothing to cut");
+ Preconditions.checkState(pastInputLengths.size() > 0,
+ "no past input to cut");
+ Preconditions.checkState(pastInputValues.length() >= minimalCutLength,
+ String.format("cannot cut %d, could cut at most %d",
+ pastInputValues.length(), minimalCutLength));
+
+ int lengthCut = minimalCutLength;
+ while (lengthCut > 0) {
+ int actualCutLength = pastInputLengths.poll();
+ pastInputValues.delete(0, actualCutLength);
+ lengthCut -= actualCutLength;
+ }
+ }
+
+ private void checkAndCut(final int inputLength) {
+ Preconditions.checkArgument(inputLength <= capacity,
+ String.format("cannot append with length %d, above capacity %d",
+ inputLength, capacity));
+
+ if (shouldCut(inputLength)) {
+ cutAtLeast(lengthToCut(inputLength));
+ }
+ }
+
+ /**
+ * Appends the specified string to this BoundedAppender, considering
+ * {@link #capacity} as upper bound in a way that former {@code append}ed
+ * whole strings are {@link StringBuilder#delete(int, int) delete}d in FIFO
+ * order.
+ *
+ * If {@code csq} is {@code null}, then the four characters {@code "null"} are
+ * considered for appending.
+ *
+ * {@code start} and {@code end} indexes are applied to {@code csq} in the
+ * general meaning of {@see CharSequence#subSequence(start, end)}, with the
+ * only exception that we do not allow empty subsequences to be appended:
+ * {@code start < end} should be held in all circumstances.
+ *
+ * @param csq The character sequence from which a subsequence will be
+ * appended. If csq is null, then characters will
+ * be appended as if csq contained the four characters
+ * "null".
+ * @param start The index of the first character in the subsequence
+ * @param end end index, excluded
+ * @return {@code this}
+ * @throws IllegalArgumentException if {@code csq} doesn't fit into
+ * {@code capacity}, or {@code end} is not more than {@code start}
+ * @throws IndexOutOfBoundsException if start or end are
+ * negative, start is greater than end, or
+ * end is greater than csq.length()
+ * @see Appendable#append(CharSequence, int, int)
+ */
+ @Override
+ public BoundedAppender append(CharSequence csq, final int start,
+ final int end) {
+ csq = ensureNull(csq);
+
+ final int inputLength = end - start;
+ Preconditions.checkPositionIndexes(start, end, csq.length());
+ Preconditions.checkElementIndex(end - 1, csq.length());
+ Preconditions.checkState(start < end, String
+ .format("end index (%d) must be before start index (%d)", end, start));
+
+ checkAndCut(inputLength);
+ pastInputValues.append(csq, start, end);
+ pastInputLengths.add(inputLength);
+
+ return this;
+ }
+
+ /**
+ * Appends the specified character to this BoundedAppender.
+ *
+ * @param c the character to append
+ * @return {@code this}
+ */
+ @Override
+ public BoundedAppender append(final char c) {
+ final int inputLength = 1;
+
+ checkAndCut(inputLength);
+ pastInputValues.append(c);
+ pastInputLengths.add(inputLength);
+
+ return this;
+ }
+
+ /**
+ *
+ * @return
+ * @see StringBuilder#length()
+ */
+ @Override
+ public int length() {
+ return pastInputValues.length();
+ }
+
+ /**
+ *
+ * @param index
+ * @return
+ * @see StringBuilder#charAt(int)
+ */
+ @Override
+ public char charAt(final int index) {
+ return pastInputValues.charAt(index);
+ }
+
+ /**
+ *
+ * @param start
+ * @param end
+ * @return
+ * @see StringBuilder#subSequence(int, int)
+ */
+ @Override
+ public CharSequence subSequence(final int start, final int end) {
+ return pastInputValues.subSequence(start, end);
+ }
+
+ /**
+ *
+ * @return
+ * @see StringBuilder#toString()
+ */
+ @Override
+ public String toString() {
+ return pastInputValues.toString();
+ }
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1e929a8c5007e12d9d8c428880d33ea0e40390bf..5d9b9ecf5729d5c48eb5e5642561eb6589ef3e8b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3014,4 +3014,13 @@
3000
+
+
+ Defines the capacity of the diagnostics message of an application attempt, in bytes.
+ Needed because ZooKeeper's StateStore cannot accomodate any message length.
+
+ yarn.app.attempt.diagnostics.capacity.bytes
+ 65536
+
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/BoundedAppenderTest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/BoundedAppenderTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..997bf737ce3dacf02266fdb4503a9a608a9a91ab
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/BoundedAppenderTest.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.junit.Assert.assertEquals;
+
+public class BoundedAppenderTest {
+ @Rule
+ public ExpectedException expected = ExpectedException.none();
+
+ @Before
+ public void setUp() throws Exception {
+
+ }
+
+ @Test
+ public void initWithZeroCapacityThrowsException() {
+ expected.expect(IllegalArgumentException.class);
+ expected.expectMessage("capacity should be positive");
+
+ new BoundedAppender(0);
+ }
+
+ @Test
+ public void initWithPositiveCapacitySuccess() {
+ new BoundedAppender(1);
+ }
+
+ @Test
+ public void nothingAppendedNothingRead() {
+ final BoundedAppender boundedAppender = new BoundedAppender();
+
+ assertEquals("nothing appended, nothing read", "",
+ boundedAppender.toString());
+ }
+
+ @Test
+ public void nullAppendedNullStringRead() {
+ final BoundedAppender boundedAppender = new BoundedAppender();
+ boundedAppender.append(null);
+
+ assertEquals("null appended, \"null\" read", "null",
+ boundedAppender.toString());
+ }
+
+ @Test
+ public void tryToAppendAboveCapacityThrowsException() {
+ final BoundedAppender boundedAppender = new BoundedAppender(1);
+
+ expected.expect(IllegalArgumentException.class);
+ expected.expectMessage("cannot append with length 2, above capacity 1");
+
+ boundedAppender.append("ab");
+ }
+
+ @Test
+ public void appendBelowCapacityOnceValueIsReadCorrectly() {
+ final BoundedAppender boundedAppender = new BoundedAppender(2);
+
+ boundedAppender.append("ab");
+
+ assertEquals("value appended is read correctly", "ab",
+ boundedAppender.toString());
+ }
+
+ @Test
+ public void appendValuesBelowCapacityAreReadCorrectlyInFifoOrder() {
+ final BoundedAppender boundedAppender = new BoundedAppender(3);
+
+ boundedAppender.append("ab");
+ boundedAppender.append("cd");
+ boundedAppender.append("e");
+ boundedAppender.append("fg");
+
+ assertEquals("last values appended fitting capacity are read correctly",
+ "efg", boundedAppender.toString());
+ }
+
+ @Test
+ public void appendValuesWithInvalidIndicesThrowsException() {
+ final BoundedAppender boundedAppender = new BoundedAppender(1);
+
+ expected.expect(IndexOutOfBoundsException.class);
+
+ boundedAppender.append("a", -1, 0);
+
+ expected.expect(IllegalStateException.class);
+
+ boundedAppender.append("a", 0, 0);
+
+ expected.expect(IndexOutOfBoundsException.class);
+
+ boundedAppender.append("a", 0, 2);
+
+ expected.expect(IndexOutOfBoundsException.class);
+
+ boundedAppender.append("a", 1, 1);
+ }
+
+ @Test
+ public void nullAppendedWithValidIndicesNullRead() {
+ final BoundedAppender boundedAppender = new BoundedAppender(4);
+
+ boundedAppender.append(null, 0, 4);
+
+ assertEquals("null appended with valid indices, \"null\" read", "null",
+ boundedAppender.toString());
+ }
+
+ @Test
+ public void appendValueWithValidIndicesIsReadCorrectly() {
+ final BoundedAppender boundedAppender = new BoundedAppender(2);
+
+ boundedAppender.append("abcd", 1, 3);
+
+ assertEquals("value appended with valid indices is read correctly", "bc",
+ boundedAppender.toString());
+ }
+
+ @Test
+ public void appendValuesWithValidIndicesAreReadCorrectlyInFifoOrder() {
+ final BoundedAppender boundedAppender = new BoundedAppender(3);
+
+ boundedAppender.append("abcd", 1, 3);
+ boundedAppender.append("efgh", 0, 2);
+ boundedAppender.append("i", 0, 1);
+ boundedAppender.append("jk", 0, 2);
+
+ assertEquals(
+ "last values appended with valid parameters fitting capacity are "
+ + "read correctly",
+ "ijk", boundedAppender.toString());
+ }
+
+ @Test
+ public void appendOneCharIsReadCorrectly() {
+ final BoundedAppender boundedAppender = new BoundedAppender(1);
+
+ boundedAppender.append('a');
+
+ assertEquals("one char appended is read correctly", "a",
+ boundedAppender.toString());
+ }
+
+ @Test
+ public void appendMultipleCharsAreReadCorrectlyInFifoOrder() {
+ final BoundedAppender boundedAppender = new BoundedAppender(2);
+
+ boundedAppender.append('a');
+ boundedAppender.append('b');
+ boundedAppender.append('c');
+ boundedAppender.append('d');
+
+ assertEquals(
+ "multiple chars appended are read correctly, the ones deleted in "
+ + "FIFO order",
+ "cd", boundedAppender.toString());
+ }
+}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index ab84985edcbc2a6ee34de90ec9de2c1edfbfcf1d..8eb5ed64110d9c50b27c1e2a4f274d4c6beaf5a8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -109,6 +109,7 @@
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.BoundedAppender;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -171,7 +172,7 @@
// Set to null initially. Will eventually get set
// if an RMAppAttemptUnregistrationEvent occurs
private FinalApplicationStatus finalStatus = null;
- private final StringBuilder diagnostics = new StringBuilder();
+ private final BoundedAppender diagnostics;
private int amContainerExitStatus = ContainerExitStatus.INVALID;
private Configuration conf;
@@ -518,6 +519,10 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
this.amReq = amReq;
this.blacklistedNodesForAM = amBlacklistManager;
+
+ this.diagnostics = new BoundedAppender(
+ conf.getInt(YarnConfiguration.APP_ATTEMPT_DIAGNOSTICS_CAPACITY_BYTES,
+ YarnConfiguration.DEFAULT_APP_ATTEMPT_DIAGNOSTICS_CAPACITY_BYTES));
}
@Override