diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FastNumberFormat.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FastNumberFormat.java index e69de29bb2d..a7ec4a1823a 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FastNumberFormat.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FastNumberFormat.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +/** + * Fast thread-safe version of NumberFormat + */ +public class FastNumberFormat { + + public static StringBuilder format(StringBuilder sb, long value, int minimumDigits) { + if (value < 0) { + sb.append('-'); + value = -value; + } + + int numDigits = 0; + long tmp = value; + do { + ++numDigits; + tmp /= 10; + } while (tmp > 0); + + for (int i = minimumDigits - numDigits; i > 0; --i) { + sb.append('0'); + } + + sb.append(value); + return sb; + } +} diff --git hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFastNumberFormat.java hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFastNumberFormat.java index e69de29bb2d..c8935dde3a5 100644 --- hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFastNumberFormat.java +++ hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFastNumberFormat.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.junit.Assert; +import org.junit.Test; + +import java.text.NumberFormat; + +/** + * Test for FastNumberFormat + */ +public class TestFastNumberFormat { + private final int MIN_DIGITS = 6; + + @Test(timeout = 1000) + public void testLongWithPadding() throws Exception { + NumberFormat numberFormat = NumberFormat.getInstance(); + numberFormat.setGroupingUsed(false); + numberFormat.setMinimumIntegerDigits(6); + long[] testLongs = {1, 23, 456, 7890, 12345, 678901, 2345689, 0, -0, -1, + -23, -456, -7890, -12345, -678901, -2345689}; + for (long l: testLongs) { + StringBuilder sb = new StringBuilder(); + FastNumberFormat.format(sb, l, MIN_DIGITS); + String fastNumberStr = sb.toString(); + Assert.assertEquals("Number formats should be equal", + numberFormat.format(l), fastNumberStr); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java index 5f3a68ebe1a..649c47d090c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java @@ -18,18 +18,15 @@ package org.apache.hadoop.yarn.api.records; -import java.text.NumberFormat; -import java.util.Iterator; import java.util.NoSuchElementException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.FastNumberFormat; import org.apache.hadoop.yarn.util.Records; -import com.google.common.base.Splitter; - /** *

ApplicationAttemptId denotes the particular attempt * of an ApplicationMaster for a given {@link ApplicationId}.

@@ -42,7 +39,6 @@ @Stable public abstract class ApplicationAttemptId implements Comparable { - private static Splitter _spliter = Splitter.on('_').trimResults(); @Private @Unstable @@ -84,16 +80,8 @@ public static ApplicationAttemptId newInstance(ApplicationId appId, @Unstable protected abstract void setAttemptId(int attemptId); - static final ThreadLocal attemptIdFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(6); - return fmt; - } - }; + private static final int ATTEMPT_ID_MIN_DIGITS = 6; + private static final int APP_ID_MIN_DIGITS = 4; @Override public int hashCode() { @@ -135,12 +123,15 @@ public int compareTo(ApplicationAttemptId other) { @Override public String toString() { - StringBuilder sb = new StringBuilder(appAttemptIdStrPrefix); + StringBuilder sb = new StringBuilder(64); + sb.append(appAttemptIdStrPrefix); + sb.append("_"); + ApplicationId appId = getApplicationId(); + sb.append(appId.getClusterTimestamp()); + sb.append("_"); + FastNumberFormat.format(sb, appId.getId(), APP_ID_MIN_DIGITS); sb.append("_"); - sb.append(this.getApplicationId().getClusterTimestamp()).append("_"); - sb.append(ApplicationId.appIdFormat.get().format( - this.getApplicationId().getId())); - sb.append("_").append(attemptIdFormat.get().format(getAttemptId())); + FastNumberFormat.format(sb, getAttemptId(), ATTEMPT_ID_MIN_DIGITS); return sb.toString(); } @@ -148,29 +139,29 @@ public String toString() { @Public @Stable - public static ApplicationAttemptId fromString(String applicationAttemptIdStr) { - Iterator it = _spliter.split(applicationAttemptIdStr).iterator(); - if (!it.next().equals(appAttemptIdStrPrefix)) { + public static ApplicationAttemptId fromString(String appAttemptIdStr) { + int pos1 = appAttemptIdStr.indexOf("_"); + String prefix = appAttemptIdStr.substring(0, pos1); + if (!appAttemptIdStrPrefix.equals(prefix)) { throw new IllegalArgumentException("Invalid AppAttemptId prefix: " - + applicationAttemptIdStr); + + appAttemptIdStr); } try { - return toApplicationAttemptId(it); + int pos2 = appAttemptIdStr.indexOf("_", pos1 + 1); + long rmId = Long.parseLong(appAttemptIdStr.substring(pos1 + 1, pos2)); + int pos3 = appAttemptIdStr.indexOf("_", pos2 + 1); + int appId = Integer.parseInt(appAttemptIdStr.substring(pos2 + 1, pos3)); + ApplicationId applicationId = ApplicationId.newInstance(rmId, appId); + int attemptId = Integer.parseInt(appAttemptIdStr.substring(pos3 + 1)); + ApplicationAttemptId applicationAttemptId = + ApplicationAttemptId.newInstance(applicationId, attemptId); + return applicationAttemptId; } catch (NumberFormatException n) { throw new IllegalArgumentException("Invalid AppAttemptId: " - + applicationAttemptIdStr, n); + + appAttemptIdStr, n); } catch (NoSuchElementException e) { throw new IllegalArgumentException("Invalid AppAttemptId: " - + applicationAttemptIdStr, e); + + appAttemptIdStr, e); } } - - private static ApplicationAttemptId toApplicationAttemptId( - Iterator it) throws NumberFormatException { - ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), - Integer.parseInt(it.next())); - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next())); - return appAttemptId; - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java index 03a77ce309f..f6f61b3338b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationId.java @@ -18,18 +18,15 @@ package org.apache.hadoop.yarn.api.records; -import java.text.NumberFormat; -import java.util.Iterator; import java.util.NoSuchElementException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.FastNumberFormat; import org.apache.hadoop.yarn.util.Records; -import com.google.common.base.Splitter; - /** *

ApplicationId represents the globally unique * identifier for an application.

@@ -42,7 +39,6 @@ @Public @Stable public abstract class ApplicationId implements Comparable { - private static Splitter _spliter = Splitter.on('_').trimResults(); @Private @Unstable @@ -87,16 +83,7 @@ public static ApplicationId newInstance(long clusterTimestamp, int id) { protected abstract void build(); - static final ThreadLocal appIdFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(4); - return fmt; - } - }; + private static final int APP_ID_MIN_DIGITS = 4; @Override public int compareTo(ApplicationId other) { @@ -110,28 +97,31 @@ public int compareTo(ApplicationId other) { @Override public String toString() { - return appIdStrPrefix + "_" + this.getClusterTimestamp() + "_" + appIdFormat - .get().format(getId()); - } - - private static ApplicationId toApplicationId( - Iterator it) throws NumberFormatException { - ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), - Integer.parseInt(it.next())); - return appId; + StringBuilder sb = new StringBuilder(64); + sb.append(appIdStrPrefix); + sb.append("_"); + sb.append(getClusterTimestamp()); + sb.append("_"); + FastNumberFormat.format(sb, getId(), APP_ID_MIN_DIGITS); + return sb.toString(); } @Public @Stable public static ApplicationId fromString(String appIdStr) { - Iterator it = _spliter.split((appIdStr)).iterator(); - if (!it.next().equals(appIdStrPrefix)) { + int pos1 = appIdStr.indexOf("_"); + String prefix = appIdStr.substring(0, pos1); + if (!appIdStrPrefix.equals(prefix)) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + appIdStrPrefix); } try { - return toApplicationId(it); + int pos2 = appIdStr.indexOf("_", pos1 + 1); + long rmId = Long.parseLong(appIdStr.substring(pos1 + 1, pos2)); + int appId = Integer.parseInt(appIdStr.substring(pos2 + 1)); + ApplicationId applicationId = ApplicationId.newInstance(rmId, appId); + return applicationId; } catch (NumberFormatException n) { throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr, n); @@ -140,7 +130,6 @@ public static ApplicationId fromString(String appIdStr) { + appIdStr, e); } } - @Override public int hashCode() { // Generated by eclipse. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java index feddeca9e70..28c9b2b0bd7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java @@ -18,16 +18,13 @@ package org.apache.hadoop.yarn.api.records; -import com.google.common.base.Splitter; - -import java.text.NumberFormat; -import java.util.Iterator; import java.util.NoSuchElementException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.FastNumberFormat; import org.apache.hadoop.yarn.util.Records; /** @@ -38,7 +35,6 @@ @Stable public abstract class ContainerId implements Comparable{ public static final long CONTAINER_ID_BITMASK = 0xffffffffffL; - private static final Splitter _SPLITTER = Splitter.on('_').trimResults(); private static final String CONTAINER_PREFIX = "container"; private static final String EPOCH_PREFIX = "e"; @@ -115,29 +111,13 @@ public static ContainerId newInstance(ApplicationAttemptId appAttemptId, protected abstract void setContainerId(long id); - // TODO: fail the app submission if attempts are more than 10 or something - private static final ThreadLocal appAttemptIdAndEpochFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(2); - return fmt; - } - }; - // TODO: Why thread local? - // ^ NumberFormat instances are not threadsafe - private static final ThreadLocal containerIdFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(6); - return fmt; - } - }; + private static final int APP_ID_MIN_DIGITS = 4; + + private static final int ATTEMPT_ID_MIN_DIGITS = 2; + + private static final int EPOCH_MIN_DIGITS = 2; + + private static final int CONTAINER_ID_MIN_DIGITS = 6; @Override public int hashCode() { @@ -185,49 +165,66 @@ public int compareTo(ContainerId other) { */ @Override public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(CONTAINER_PREFIX + "_"); + StringBuilder sb = new StringBuilder(64); + sb.append(CONTAINER_PREFIX); + sb.append("_"); long epoch = getContainerId() >> 40; if (epoch > 0) { - sb.append(EPOCH_PREFIX) - .append(appAttemptIdAndEpochFormat.get().format(epoch)).append("_");; + sb.append(EPOCH_PREFIX); + FastNumberFormat.format(sb, epoch, EPOCH_MIN_DIGITS); + sb.append("_"); } ApplicationId appId = getApplicationAttemptId().getApplicationId(); - sb.append(appId.getClusterTimestamp()).append("_"); - sb.append(ApplicationId.appIdFormat.get().format(appId.getId())) - .append("_"); - sb.append( - appAttemptIdAndEpochFormat.get().format( - getApplicationAttemptId().getAttemptId())).append("_"); - sb.append(containerIdFormat.get() - .format(CONTAINER_ID_BITMASK & getContainerId())); + sb.append(appId.getClusterTimestamp()); + sb.append("_"); + FastNumberFormat.format(sb, appId.getId(), APP_ID_MIN_DIGITS); + sb.append("_"); + FastNumberFormat.format(sb, getApplicationAttemptId().getAttemptId(), + ATTEMPT_ID_MIN_DIGITS); + sb.append("_"); + FastNumberFormat.format(sb, CONTAINER_ID_BITMASK & getContainerId(), + CONTAINER_ID_MIN_DIGITS); return sb.toString(); } @Public @Stable public static ContainerId fromString(String containerIdStr) { - Iterator it = _SPLITTER.split(containerIdStr).iterator(); - if (!it.next().equals(CONTAINER_PREFIX)) { + int pos1 = containerIdStr.indexOf("_"); + String prefix = containerIdStr.substring(0, pos1); + if (!CONTAINER_PREFIX.equals(prefix)) { throw new IllegalArgumentException("Invalid ContainerId prefix: " + containerIdStr); } try { - String epochOrClusterTimestampStr = it.next(); + int pos2 = containerIdStr.indexOf("_", pos1 + 1); + String epochOrClusterTimestampStr = + containerIdStr.substring(pos1 + 1, pos2); long epoch = 0; - ApplicationAttemptId appAttemptID = null; + String clusterTimestampStr; if (epochOrClusterTimestampStr.startsWith(EPOCH_PREFIX)) { String epochStr = epochOrClusterTimestampStr; epoch = Integer.parseInt(epochStr.substring(EPOCH_PREFIX.length())); - appAttemptID = toApplicationAttemptId(it); + int pos3 = containerIdStr.indexOf("_", pos2 + 1); + clusterTimestampStr = containerIdStr.substring(pos2 + 1, pos3); + pos2 = pos3; } else { - String clusterTimestampStr = epochOrClusterTimestampStr; - long clusterTimestamp = Long.parseLong(clusterTimestampStr); - appAttemptID = toApplicationAttemptId(clusterTimestamp, it); + clusterTimestampStr = epochOrClusterTimestampStr; } - long id = Long.parseLong(it.next()); + long clusterTimestamp = Long.parseLong(clusterTimestampStr); + + int pos3 = containerIdStr.indexOf("_", pos2 + 1); + int appId = Integer.parseInt(containerIdStr.substring(pos2 + 1, pos3)); + ApplicationId applicationId = ApplicationId.newInstance(clusterTimestamp, + appId); + int pos4 = containerIdStr.indexOf("_", pos3 + 1); + int attemptId = Integer.parseInt( + containerIdStr.substring(pos3 + 1, pos4)); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(applicationId, attemptId); + long id = Long.parseLong(containerIdStr.substring(pos4 + 1)); long cid = (epoch << 40) | id; - ContainerId containerId = ContainerId.newContainerId(appAttemptID, cid); + ContainerId containerId = ContainerId.newContainerId(appAttemptId, cid); return containerId; } catch (NumberFormatException n) { throw new IllegalArgumentException("Invalid ContainerId: " @@ -238,19 +235,5 @@ public static ContainerId fromString(String containerIdStr) { } } - private static ApplicationAttemptId toApplicationAttemptId( - Iterator it) throws NumberFormatException { - return toApplicationAttemptId(Long.parseLong(it.next()), it); - } - - private static ApplicationAttemptId toApplicationAttemptId( - long clusterTimestamp, Iterator it) throws NumberFormatException { - ApplicationId appId = ApplicationId.newInstance(clusterTimestamp, - Integer.parseInt(it.next())); - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next())); - return appAttemptId; - } - protected abstract void build(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java index 71474b14b88..38ba6d68d14 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationId.java @@ -19,11 +19,11 @@ package org.apache.hadoop.yarn.api.records; import java.io.IOException; -import java.text.NumberFormat; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.FastNumberFormat; import org.apache.hadoop.yarn.util.Records; /** @@ -89,16 +89,7 @@ public static ReservationId newInstance(long clusterTimestamp, long id) { protected abstract void build(); - static final ThreadLocal reservIdFormat = - new ThreadLocal() { - @Override - public NumberFormat initialValue() { - NumberFormat fmt = NumberFormat.getInstance(); - fmt.setGroupingUsed(false); - fmt.setMinimumIntegerDigits(4); - return fmt; - } - }; + private static final int RESERVATION_ID_MIN_DIGITS = 4; @Override public int compareTo(ReservationId other) { @@ -112,8 +103,12 @@ public int compareTo(ReservationId other) { @Override public String toString() { - return reserveIdStrPrefix + this.getClusterTimestamp() + "_" - + reservIdFormat.get().format(getId()); + StringBuilder sb = new StringBuilder(64); + sb.append(reserveIdStrPrefix); + sb.append(getClusterTimestamp()); + sb.append("_"); + FastNumberFormat.format(sb, getId(), RESERVATION_ID_MIN_DIGITS); + return sb.toString(); } /**