From 30804d2a5bd8ac068ddb7f4da5cf78867a33cd38 Mon Sep 17 00:00:00 2001 From: Yi Deng Date: Tue, 28 Oct 2014 14:34:47 -0700 Subject: [PATCH] Add `Percentiles` who uses `FatLongHistogram` to replace `MultableHistogram` Summary: Move `FastLongHistogram` and `AtomicUtils` from `hbase-common` to `hbase-hadoop2-comp` package Create the method `DynamicMetricsRegistry.newPercentiles` and replace using of `newHistogram` with it. Test Plan: `TestPercentiles` `TestAtomicUtils` Differential Revision: https://reviews.facebook.net/D27669 --- .../org/apache/hadoop/hbase/util/AtomicUtils.java | 63 ----- .../hadoop/hbase/util/FastLongHistogram.java | 233 ---------------- .../hadoop/hbase/util/TestFastLongHistogram.java | 100 ------- .../hbase/ipc/MetricsHBaseServerSourceImpl.java | 10 +- .../master/MetricsAssignmentManagerSourceImpl.java | 14 +- .../master/MetricsMasterFilesystemSourceImpl.java | 18 +- .../hbase/master/MetricsSnapshotSourceImpl.java | 16 +- .../master/balancer/MetricsBalancerSourceImpl.java | 6 +- .../MetricsRegionServerSourceImpl.java | 26 +- .../regionserver/MetricsRegionSourceImpl.java | 10 +- .../wal/MetricsEditsReplaySourceImpl.java | 20 +- .../regionserver/wal/MetricsWALSourceImpl.java | 17 +- .../thrift/MetricsThriftServerSourceImpl.java | 21 +- .../org/apache/hadoop/hbase/util/AtomicUtils.java | 63 +++++ .../hadoop/hbase/util/FastLongHistogram.java | 303 +++++++++++++++++++++ .../org/apache/hadoop/hbase/util/Percentiles.java | 194 +++++++++++++ .../metrics2/lib/DynamicMetricsRegistry.java | 6 + .../apache/hadoop/hbase/util/TestAtomicUtils.java | 52 ++++ .../hadoop/hbase/util/TestFastLongHistogram.java | 100 +++++++ .../apache/hadoop/hbase/util/TestPercentiles.java | 74 +++++ 20 files changed, 873 insertions(+), 473 deletions(-) delete mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java delete mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java delete mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java create mode 100644 hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java create mode 100644 hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java create mode 100644 hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/Percentiles.java create mode 100644 hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestAtomicUtils.java create mode 100644 hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java create mode 100644 hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestPercentiles.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java deleted file mode 100644 index 35391ee..0000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * Utilities related to atomic operations. - */ -@InterfaceAudience.Private -public class AtomicUtils { - /** - * Updates a AtomicLong which is supposed to maintain the minimum values. This method is not - * synchronized but is thread-safe. - */ - public static void updateMin(AtomicLong min, long value) { - while (true) { - long cur = min.get(); - if (value >= cur) { - break; - } - - if (min.compareAndSet(cur, value)) { - break; - } - } - } - - /** - * Updates a AtomicLong which is supposed to maintain the maximum values. This method is not - * synchronized but is thread-safe. - */ - public static void updateMax(AtomicLong max, long value) { - while (true) { - long cur = max.get(); - if (value <= cur) { - break; - } - - if (max.compareAndSet(cur, value)) { - break; - } - } - } - -} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java deleted file mode 100644 index 623cbdb..0000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java +++ /dev/null @@ -1,233 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicLongArray; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -/** - * FastLongHistogram is a thread-safe class that estimate distribution of data and computes the - * quantiles. - */ -@InterfaceAudience.Public -@InterfaceStability.Evolving -public class FastLongHistogram { - /** - * Bins is a class containing a list of buckets(or bins) for estimation histogram of some data. - */ - private static class Bins { - private final AtomicLongArray counts; - // inclusive - private final long binsMin; - // exclusive - private final long binsMax; - private final long bins10XMax; - private final AtomicLong min = new AtomicLong(Long.MAX_VALUE); - private final AtomicLong max = new AtomicLong(0L); - // set to true when any of data has been inserted to the Bins. It is set after the counts are - // updated. - private final AtomicBoolean hasData = new AtomicBoolean(false); - - /** - * The constructor for creating a Bins without any prior data. - */ - public Bins() { - this.counts = new AtomicLongArray(4); - this.binsMin = 0L; - this.binsMax = Long.MAX_VALUE; - this.bins10XMax = Long.MAX_VALUE; - } - - /** - * The constructor for creating a Bins with last Bins. - * @param last the last Bins instance. - * @param quantiles the quantiles for creating the bins of the histogram. - */ - public Bins(Bins last, int numOfBins, double minQ, double maxQ) { - long[] values = last.getQuantiles(new double[] { minQ, maxQ }); - long wd = values[1] - values[0] + 1; - // expand minQ and maxQ in two ends back assuming uniform distribution - this.binsMin = Math.max(0L, (long) (values[0] - wd * minQ)); - long binsMax = (long) (values[1] + wd * (1 - maxQ)) + 1; - // make sure each of bins is at least of width 1 - this.binsMax = Math.max(binsMax, this.binsMin + numOfBins); - this.bins10XMax = Math.max((long) (values[1] + (binsMax - 1) * 9), this.binsMax + 1); - - this.counts = new AtomicLongArray(numOfBins + 3); - } - - /** - * Adds a value to the histogram. - */ - public void add(long value, long count) { - AtomicUtils.updateMin(min, value); - AtomicUtils.updateMax(max, value); - - if (value < this.binsMin) { - this.counts.addAndGet(0, count); - } else if (value > this.bins10XMax) { - this.counts.addAndGet(this.counts.length() - 1, count); - } else if (value >= this.binsMax) { - this.counts.addAndGet(this.counts.length() - 2, count); - } else { - // compute the position - int pos = - 1 + (int) ((value - this.binsMin) * (this.counts.length() - 3) / (this.binsMax - this.binsMin)); - this.counts.addAndGet(pos, count); - } - - // hasData needs to be updated as last - this.hasData.set(true); - } - - /** - * Computes the quantiles give the ratios. - * @param smooth set to true to have a prior on the distribution. Used for recreating the bins. - */ - public long[] getQuantiles(double[] quantiles) { - if (!this.hasData.get()) { - // No data yet. - return new long[quantiles.length]; - } - - // Make a snapshot of lowerCounter, higherCounter and bins.counts to counts. - // This is not synchronized, but since the counter are accumulating, the result is a good - // estimation of a snapshot. - long[] counts = new long[this.counts.length()]; - long total = 0L; - for (int i = 0; i < this.counts.length(); i++) { - counts[i] = this.counts.get(i); - total += counts[i]; - } - - int rIndex = 0; - double qCount = total * quantiles[0]; - long cum = 0L; - - long[] res = new long[quantiles.length]; - countsLoop: for (int i = 0; i < counts.length; i++) { - // mn and mx define a value range - long mn, mx; - if (i == 0) { - mn = this.min.get(); - mx = this.binsMin; - } else if (i == counts.length - 1) { - mn = this.bins10XMax; - mx = this.max.get(); - } else if (i == counts.length - 2) { - mn = this.binsMax; - mx = this.bins10XMax; - } else { - mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length() - 3); - mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length() - 3); - } - - if (mx < this.min.get()) { - continue; - } - if (mn > this.max.get()) { - break; - } - mn = Math.max(mn, this.min.get()); - mx = Math.min(mx, this.max.get()); - - // lastCum/cum are the corresponding counts to mn/mx - double lastCum = cum; - cum += counts[i]; - - // fill the results for qCount is within current range. - while (qCount <= cum) { - if (cum == lastCum) { - res[rIndex] = mn; - } else { - res[rIndex] = (long) ((qCount - lastCum) * (mx - mn) / (cum - lastCum) + mn); - } - - // move to next quantile - rIndex++; - if (rIndex >= quantiles.length) { - break countsLoop; - } - qCount = total * quantiles[rIndex]; - } - } - // In case quantiles contains values >= 100% - for (; rIndex < quantiles.length; rIndex++) { - res[rIndex] = this.max.get(); - } - - return res; - } - } - - // The bins counting values. It is replaced with a new one in calling of reset(). - private volatile Bins bins = new Bins(); - // The quantiles for creating a Bins with last Bins. - private final int numOfBins; - - /** - * Constructor. - * @param numOfBins the number of bins for the histogram. A larger value results in more precise - * results but with lower efficiency, and vice versus. - */ - public FastLongHistogram(int numOfBins) { - this.numOfBins = numOfBins; - } - - /** - * Constructor setting the bins assuming a uniform distribution within a range. - * @param numOfBins the number of bins for the histogram. A larger value results in more precise - * results but with lower efficiency, and vice versus. - * @param min lower bound of the region, inclusive. - * @param max higher bound of the region, inclusive. - */ - public FastLongHistogram(int numOfBins, long min, long max) { - this(numOfBins); - Bins bins = new Bins(); - bins.add(min, 1); - bins.add(max, 1); - this.bins = new Bins(bins, numOfBins, 0.01, 0.99); - } - - /** - * Adds a value to the histogram. - */ - public void add(long value, long count) { - this.bins.add(value, count); - } - - /** - * Computes the quantiles give the ratios. - */ - public long[] getQuantiles(double[] quantiles) { - return this.bins.getQuantiles(quantiles); - } - - /** - * Resets the histogram for new counting. - */ - public void reset() { - if (this.bins.hasData.get()) { - this.bins = new Bins(this.bins, numOfBins, 0.01, 0.99); - } - } -} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java deleted file mode 100644 index f5848f3..0000000 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import java.util.Arrays; -import java.util.Random; - -import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Testcases for FastLongHistogram. - */ -@Category({MiscTests.class, SmallTests.class}) -public class TestFastLongHistogram { - - private static void doTestUniform(FastLongHistogram hist) { - long[] VALUES = { 0, 10, 20, 30, 40, 50 }; - double[] qs = new double[VALUES.length]; - for (int i = 0; i < qs.length; i++) { - qs[i] = (double) VALUES[i] / VALUES[VALUES.length - 1]; - } - - for (int i = 0; i < 10; i++) { - for (long v : VALUES) { - hist.add(v, 1); - } - long[] vals = hist.getQuantiles(qs); - System.out.println(Arrays.toString(vals)); - for (int j = 0; j < qs.length; j++) { - Assert.assertTrue(j + "-th element org: " + VALUES[j] + ", act: " + vals[j], - Math.abs(vals[j] - VALUES[j]) <= 10); - } - hist.reset(); - } - } - - @Test - public void testUniform() { - FastLongHistogram hist = new FastLongHistogram(100, 0, 50); - doTestUniform(hist); - } - - @Test - public void testAdaptionOfChange() { - // assumes the uniform distribution - FastLongHistogram hist = new FastLongHistogram(100, 0, 100); - - Random rand = new Random(); - - for (int n = 0; n < 10; n++) { - for (int i = 0; i < 900; i++) { - hist.add(rand.nextInt(100), 1); - } - - // add 10% outliers, this breaks the assumption, hope bin10xMax works - for (int i = 0; i < 100; i++) { - hist.add(1000 + rand.nextInt(100), 1); - } - - long[] vals = hist.getQuantiles(new double[] { 0.25, 0.75, 0.95 }); - System.out.println(Arrays.toString(vals)); - if (n == 0) { - Assert.assertTrue("Out of possible value", vals[0] >= 0 && vals[0] <= 50); - Assert.assertTrue("Out of possible value", vals[1] >= 50 && vals[1] <= 100); - Assert.assertTrue("Out of possible value", vals[2] >= 900 && vals[2] <= 1100); - } - - hist.reset(); - } - } - - @Test - public void testSameValues() { - FastLongHistogram hist = new FastLongHistogram(100); - - hist.add(50, 100); - - hist.reset(); - doTestUniform(hist); - } -} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java index 2f5e5cf..6bfbb7e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java @@ -21,11 +21,11 @@ package org.apache.hadoop.hbase.ipc; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.hbase.util.Percentiles; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.Interns; import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; @InterfaceAudience.Private public class MetricsHBaseServerSourceImpl extends BaseSourceImpl @@ -38,8 +38,8 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl private final MutableCounterLong authenticationFailures; private final MutableCounterLong sentBytes; private final MutableCounterLong receivedBytes; - private MutableHistogram queueCallTime; - private MutableHistogram processCallTime; + private Percentiles queueCallTime; + private Percentiles processCallTime; public MetricsHBaseServerSourceImpl(String metricsName, String metricsDescription, @@ -62,9 +62,9 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl SENT_BYTES_DESC, 0l); this.receivedBytes = this.getMetricsRegistry().newCounter(RECEIVED_BYTES_NAME, RECEIVED_BYTES_DESC, 0l); - this.queueCallTime = this.getMetricsRegistry().newHistogram(QUEUE_CALL_TIME_NAME, + this.queueCallTime = this.getMetricsRegistry().newPercentiles(QUEUE_CALL_TIME_NAME, QUEUE_CALL_TIME_DESC); - this.processCallTime = this.getMetricsRegistry().newHistogram(PROCESS_CALL_TIME_NAME, + this.processCallTime = this.getMetricsRegistry().newPercentiles(PROCESS_CALL_TIME_NAME, PROCESS_CALL_TIME_DESC); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java index 215855f..0f7b5dd 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java @@ -20,8 +20,8 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.hbase.util.Percentiles; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; @InterfaceAudience.Private public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implements MetricsAssignmentManagerSource { @@ -29,8 +29,8 @@ public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implement private MutableGaugeLong ritGauge; private MutableGaugeLong ritCountOverThresholdGauge; private MutableGaugeLong ritOldestAgeGauge; - private MutableHistogram assignTimeHisto; - private MutableHistogram bulkAssignTimeHisto; + private Percentiles assignTimeHisto; + private Percentiles bulkAssignTimeHisto; public MetricsAssignmentManagerSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); @@ -42,12 +42,13 @@ public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implement super(metricsName, metricsDescription, metricsContext, metricsJmxContext); } + @Override public void init() { ritGauge = metricsRegistry.newGauge(RIT_COUNT_NAME, "", 0l); ritCountOverThresholdGauge = metricsRegistry.newGauge(RIT_COUNT_OVER_THRESHOLD_NAME, "", 0l); ritOldestAgeGauge = metricsRegistry.newGauge(RIT_OLDEST_AGE_NAME, "", 0l); - assignTimeHisto = metricsRegistry.newHistogram(ASSIGN_TIME_NAME); - bulkAssignTimeHisto = metricsRegistry.newHistogram(BULK_ASSIGN_TIME_NAME); + assignTimeHisto = metricsRegistry.newPercentiles(ASSIGN_TIME_NAME, ""); + bulkAssignTimeHisto = metricsRegistry.newPercentiles(BULK_ASSIGN_TIME_NAME, ""); } @Override @@ -60,14 +61,17 @@ public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implement bulkAssignTimeHisto.add(time); } + @Override public void setRIT(int ritCount) { ritGauge.set(ritCount); } + @Override public void setRITCountOverThreshold(int ritCount) { ritCountOverThresholdGauge.set(ritCount); } + @Override public void setRITOldestAge(long ritCount) { ritOldestAgeGauge.set(ritCount); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java index 1644207..de3f2f6 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java @@ -20,15 +20,15 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.hbase.util.Percentiles; @InterfaceAudience.Private public class MetricsMasterFilesystemSourceImpl extends BaseSourceImpl implements MetricsMasterFileSystemSource { - private MutableHistogram splitSizeHisto; - private MutableHistogram splitTimeHisto; - private MutableHistogram metaSplitTimeHisto; - private MutableHistogram metaSplitSizeHisto; + private Percentiles splitSizeHisto; + private Percentiles splitTimeHisto; + private Percentiles metaSplitTimeHisto; + private Percentiles metaSplitSizeHisto; public MetricsMasterFilesystemSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); @@ -42,10 +42,10 @@ public class MetricsMasterFilesystemSourceImpl extends BaseSourceImpl implements @Override public void init() { - splitSizeHisto = metricsRegistry.newHistogram(SPLIT_SIZE_NAME, SPLIT_SIZE_DESC); - splitTimeHisto = metricsRegistry.newHistogram(SPLIT_TIME_NAME, SPLIT_TIME_DESC); - metaSplitTimeHisto = metricsRegistry.newHistogram(META_SPLIT_TIME_NAME, META_SPLIT_TIME_DESC); - metaSplitSizeHisto = metricsRegistry.newHistogram(META_SPLIT_SIZE_NAME, META_SPLIT_SIZE_DESC); + splitSizeHisto = metricsRegistry.newPercentiles(SPLIT_SIZE_NAME, SPLIT_SIZE_DESC); + splitTimeHisto = metricsRegistry.newPercentiles(SPLIT_TIME_NAME, SPLIT_TIME_DESC); + metaSplitTimeHisto = metricsRegistry.newPercentiles(META_SPLIT_TIME_NAME, META_SPLIT_TIME_DESC); + metaSplitSizeHisto = metricsRegistry.newPercentiles(META_SPLIT_SIZE_NAME, META_SPLIT_SIZE_DESC); } @Override diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java index 80e1bad..9045489 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java @@ -20,14 +20,14 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.hbase.util.Percentiles; @InterfaceAudience.Private public class MetricsSnapshotSourceImpl extends BaseSourceImpl implements MetricsSnapshotSource { - private MutableHistogram snapshotTimeHisto; - private MutableHistogram snapshotCloneTimeHisto; - private MutableHistogram snapshotRestoreTimeHisto; + private Percentiles snapshotTimeHisto; + private Percentiles snapshotCloneTimeHisto; + private Percentiles snapshotRestoreTimeHisto; public MetricsSnapshotSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); @@ -41,11 +41,13 @@ public class MetricsSnapshotSourceImpl extends BaseSourceImpl implements Metrics @Override public void init() { - snapshotTimeHisto = metricsRegistry.newHistogram( + snapshotTimeHisto = metricsRegistry.newPercentiles( SNAPSHOT_TIME_NAME, SNAPSHOT_TIME_DESC); - snapshotCloneTimeHisto = metricsRegistry.newHistogram( + snapshotCloneTimeHisto = + metricsRegistry.newPercentiles( SNAPSHOT_CLONE_TIME_NAME, SNAPSHOT_CLONE_TIME_DESC); - snapshotRestoreTimeHisto = metricsRegistry.newHistogram( + snapshotRestoreTimeHisto = + metricsRegistry.newPercentiles( SNAPSHOT_RESTORE_TIME_NAME, SNAPSHOT_RESTORE_TIME_DESC); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java index 774e2e7..689cdb1 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java @@ -20,13 +20,13 @@ package org.apache.hadoop.hbase.master.balancer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.hbase.util.Percentiles; import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; @InterfaceAudience.Private public class MetricsBalancerSourceImpl extends BaseSourceImpl implements MetricsBalancerSource{ - private MutableHistogram blanceClusterHisto; + private Percentiles blanceClusterHisto; private MutableCounterLong miscCount; public MetricsBalancerSourceImpl() { @@ -42,7 +42,7 @@ public class MetricsBalancerSourceImpl extends BaseSourceImpl implements Metrics @Override public void init() { - blanceClusterHisto = metricsRegistry.newHistogram(BALANCE_CLUSTER); + blanceClusterHisto = metricsRegistry.newPercentiles(BALANCE_CLUSTER, ""); miscCount = metricsRegistry.newCounter(MISC_INVOATION_COUNT, "", 0L); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index a6377c0..50cb1d7 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.hbase.util.Percentiles; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.Interns; @@ -38,12 +38,12 @@ public class MetricsRegionServerSourceImpl final MetricsRegionServerWrapper rsWrap; - private final MetricHistogram putHisto; - private final MetricHistogram deleteHisto; - private final MetricHistogram getHisto; - private final MetricHistogram incrementHisto; - private final MetricHistogram appendHisto; - private final MetricHistogram replayHisto; + private final Percentiles putHisto; + private final Percentiles deleteHisto; + private final Percentiles getHisto; + private final Percentiles incrementHisto; + private final Percentiles appendHisto; + private final Percentiles replayHisto; private final MutableCounterLong slowPut; private final MutableCounterLong slowDelete; @@ -64,22 +64,22 @@ public class MetricsRegionServerSourceImpl super(metricsName, metricsDescription, metricsContext, metricsJmxContext); this.rsWrap = rsWrap; - putHisto = getMetricsRegistry().newHistogram(MUTATE_KEY); + putHisto = getMetricsRegistry().newPercentiles(MUTATE_KEY, ""); slowPut = getMetricsRegistry().newCounter(SLOW_MUTATE_KEY, SLOW_MUTATE_DESC, 0l); - deleteHisto = getMetricsRegistry().newHistogram(DELETE_KEY); + deleteHisto = getMetricsRegistry().newPercentiles(DELETE_KEY, ""); slowDelete = getMetricsRegistry().newCounter(SLOW_DELETE_KEY, SLOW_DELETE_DESC, 0l); - getHisto = getMetricsRegistry().newHistogram(GET_KEY); + getHisto = getMetricsRegistry().newPercentiles(GET_KEY, ""); slowGet = getMetricsRegistry().newCounter(SLOW_GET_KEY, SLOW_GET_DESC, 0l); - incrementHisto = getMetricsRegistry().newHistogram(INCREMENT_KEY); + incrementHisto = getMetricsRegistry().newPercentiles(INCREMENT_KEY, ""); slowIncrement = getMetricsRegistry().newCounter(SLOW_INCREMENT_KEY, SLOW_INCREMENT_DESC, 0l); - appendHisto = getMetricsRegistry().newHistogram(APPEND_KEY); + appendHisto = getMetricsRegistry().newPercentiles(APPEND_KEY, ""); slowAppend = getMetricsRegistry().newCounter(SLOW_APPEND_KEY, SLOW_APPEND_DESC, 0l); - replayHisto = getMetricsRegistry().newHistogram(REPLAY_KEY); + replayHisto = getMetricsRegistry().newPercentiles(REPLAY_KEY, ""); } @Override diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index df23942..1eeec5e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -24,12 +24,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.Percentiles; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.impl.JmxCacheBuster; import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; import org.apache.hadoop.metrics2.lib.Interns; import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; @InterfaceAudience.Private public class MetricsRegionSourceImpl implements MetricsRegionSource { @@ -55,8 +55,8 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { private MutableCounterLong regionIncrement; private MutableCounterLong regionAppend; - private MutableHistogram regionGet; - private MutableHistogram regionScanNext; + private Percentiles regionGet; + private Percentiles regionScanNext; public MetricsRegionSourceImpl(MetricsRegionWrapper regionWrapper, MetricsRegionAggregateSourceImpl aggregate) { @@ -89,10 +89,10 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { regionAppend = registry.getLongCounter(regionAppendKey, 0l); regionGetKey = regionNamePrefix + MetricsRegionServerSource.GET_KEY; - regionGet = registry.newHistogram(regionGetKey); + regionGet = registry.newPercentiles(regionGetKey, ""); regionScanNextKey = regionNamePrefix + MetricsRegionServerSource.SCAN_NEXT_KEY; - regionScanNext = registry.newHistogram(regionScanNextKey); + regionScanNext = registry.newPercentiles(regionScanNextKey, ""); } @Override diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsEditsReplaySourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsEditsReplaySourceImpl.java index b10a69e..f0e4b0a 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsEditsReplaySourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsEditsReplaySourceImpl.java @@ -18,11 +18,9 @@ package org.apache.hadoop.hbase.regionserver.wal; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.hbase.util.Percentiles; /** * Hadoop1 implementation of MetricsMasterSource. Implements BaseSource through BaseSourceImpl, @@ -32,11 +30,9 @@ import org.apache.hadoop.metrics2.MetricHistogram; public class MetricsEditsReplaySourceImpl extends BaseSourceImpl implements MetricsEditsReplaySource { - private static final Log LOG = LogFactory.getLog(MetricsEditsReplaySourceImpl.class.getName()); - - private MetricHistogram replayTimeHisto; - private MetricHistogram replayBatchSizeHisto; - private MetricHistogram replayDataSizeHisto; + private Percentiles replayTimeHisto; + private Percentiles replayBatchSizeHisto; + private Percentiles replayDataSizeHisto; public MetricsEditsReplaySourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); @@ -52,11 +48,11 @@ public class MetricsEditsReplaySourceImpl extends BaseSourceImpl implements @Override public void init() { super.init(); - replayTimeHisto = metricsRegistry.newHistogram(REPLAY_TIME_NAME, REPLAY_TIME_DESC); - replayBatchSizeHisto = metricsRegistry.newHistogram(REPLAY_BATCH_SIZE_NAME, + replayTimeHisto = metricsRegistry.newPercentiles(REPLAY_TIME_NAME, REPLAY_TIME_DESC); + replayBatchSizeHisto = metricsRegistry.newPercentiles(REPLAY_BATCH_SIZE_NAME, REPLAY_BATCH_SIZE_DESC); - replayDataSizeHisto = metricsRegistry - .newHistogram(REPLAY_DATA_SIZE_NAME, REPLAY_DATA_SIZE_DESC); + replayDataSizeHisto = + metricsRegistry.newPercentiles(REPLAY_DATA_SIZE_NAME, REPLAY_DATA_SIZE_DESC); } @Override diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java index ad8f24c..dd0e6f1 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.regionserver.wal; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.hbase.util.Percentiles; import org.apache.hadoop.metrics2.lib.MutableCounterLong; @@ -32,9 +32,9 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; @InterfaceAudience.Private public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSource { - private final MetricHistogram appendSizeHisto; - private final MetricHistogram appendTimeHisto; - private final MetricHistogram syncTimeHisto; + private final Percentiles appendSizeHisto; + private final Percentiles appendTimeHisto; + private final Percentiles syncTimeHisto; private final MutableCounterLong appendCount; private final MutableCounterLong slowAppendCount; @@ -49,11 +49,12 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo super(metricsName, metricsDescription, metricsContext, metricsJmxContext); //Create and store the metrics that will be used. - appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC); - appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC); + appendTimeHisto = this.getMetricsRegistry().newPercentiles(APPEND_TIME, APPEND_TIME_DESC); + appendSizeHisto = this.getMetricsRegistry().newPercentiles(APPEND_SIZE, APPEND_SIZE_DESC); appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l); - slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l); - syncTimeHisto = this.getMetricsRegistry().newHistogram(SYNC_TIME, SYNC_TIME_DESC); + slowAppendCount = + this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l); + syncTimeHisto = this.getMetricsRegistry().newPercentiles(SYNC_TIME, SYNC_TIME_DESC); } @Override diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java index e572e19..895e2c2 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.thrift; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.hbase.util.Percentiles; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.lib.MutableHistogram; @@ -32,12 +33,12 @@ import org.apache.hadoop.metrics2.lib.MutableHistogram; public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements MetricsThriftServerSource { - private MutableHistogram batchGetStat; - private MutableHistogram batchMutateStat; - private MutableHistogram queueTimeStat; + private Percentiles batchGetStat; + private Percentiles batchMutateStat; + private Percentiles queueTimeStat; - private MutableHistogram thriftCallStat; - private MutableHistogram thriftSlowCallStat; + private Percentiles thriftCallStat; + private Percentiles thriftSlowCallStat; private MutableGaugeLong callQueueLenGauge; @@ -51,11 +52,11 @@ public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements @Override public void init() { super.init(); - batchGetStat = getMetricsRegistry().newHistogram(BATCH_GET_KEY); - batchMutateStat = getMetricsRegistry().newHistogram(BATCH_MUTATE_KEY); - queueTimeStat = getMetricsRegistry().newHistogram(TIME_IN_QUEUE_KEY); - thriftCallStat = getMetricsRegistry().newHistogram(THRIFT_CALL_KEY); - thriftSlowCallStat = getMetricsRegistry().newHistogram(SLOW_THRIFT_CALL_KEY); + batchGetStat = getMetricsRegistry().newPercentiles(BATCH_GET_KEY, ""); + batchMutateStat = getMetricsRegistry().newPercentiles(BATCH_MUTATE_KEY, ""); + queueTimeStat = getMetricsRegistry().newPercentiles(TIME_IN_QUEUE_KEY, ""); + thriftCallStat = getMetricsRegistry().newPercentiles(THRIFT_CALL_KEY, ""); + thriftSlowCallStat = getMetricsRegistry().newPercentiles(SLOW_THRIFT_CALL_KEY, ""); callQueueLenGauge = getMetricsRegistry().getLongGauge(CALL_QUEUE_LEN_KEY, 0); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java new file mode 100644 index 0000000..35391ee --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * Utilities related to atomic operations. + */ +@InterfaceAudience.Private +public class AtomicUtils { + /** + * Updates a AtomicLong which is supposed to maintain the minimum values. This method is not + * synchronized but is thread-safe. + */ + public static void updateMin(AtomicLong min, long value) { + while (true) { + long cur = min.get(); + if (value >= cur) { + break; + } + + if (min.compareAndSet(cur, value)) { + break; + } + } + } + + /** + * Updates a AtomicLong which is supposed to maintain the maximum values. This method is not + * synchronized but is thread-safe. + */ + public static void updateMax(AtomicLong max, long value) { + while (true) { + long cur = max.get(); + if (value <= cur) { + break; + } + + if (max.compareAndSet(cur, value)) { + break; + } + } + } + +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java new file mode 100644 index 0000000..1c9231a --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java @@ -0,0 +1,303 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * FastLongHistogram is a thread-safe class that estimate distribution of data and computes the + * quantiles. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class FastLongHistogram { + /** + * Bins is a class containing a list of buckets(or bins) for estimation histogram of some data. + */ + private static class Bins { + private final AtomicLongArray counts; + // inclusive + private final long binsMin; + // exclusive + private final long binsMax; + private final long bins10XMax; + private final AtomicLong min = new AtomicLong(Long.MAX_VALUE); + private final AtomicLong max = new AtomicLong(0L); + // set to true when any of data has been inserted to the Bins. It is set after the counts are + // updated. + private final AtomicBoolean hasData = new AtomicBoolean(false); + + /** + * The constructor for creating a Bins without any prior data. + */ + public Bins() { + this.counts = new AtomicLongArray(4); + this.binsMin = 0L; + this.binsMax = Long.MAX_VALUE; + this.bins10XMax = Long.MAX_VALUE; + } + + /** + * The constructor for creating a Bins with last Bins. + * @param last the last Bins instance. + * @param quantiles the quantiles for creating the bins of the histogram. + */ + public Bins(Bins last, int numOfBins, double minQ, double maxQ) { + long[] values = last.getQuantiles(new double[] { minQ, maxQ }); + long wd = values[1] - values[0] + 1; + // expand minQ and maxQ in two ends back assuming uniform distribution + this.binsMin = Math.max(0L, (long) (values[0] - wd * minQ)); + long binsMax = (long) (values[1] + wd * (1 - maxQ)) + 1; + // make sure each of bins is at least of width 1 + this.binsMax = Math.max(binsMax, this.binsMin + numOfBins); + this.bins10XMax = Math.max(values[1] + (binsMax - 1) * 9, this.binsMax + 1); + + this.counts = new AtomicLongArray(numOfBins + 3); + } + + /** + * Adds a value to the histogram. + */ + public void add(long value, long count) { + AtomicUtils.updateMin(min, value); + AtomicUtils.updateMax(max, value); + + if (value < this.binsMin) { + this.counts.addAndGet(0, count); + } else if (value > this.bins10XMax) { + this.counts.addAndGet(this.counts.length() - 1, count); + } else if (value >= this.binsMax) { + this.counts.addAndGet(this.counts.length() - 2, count); + } else { + // compute the position + int pos = + 1 + (int) ((value - this.binsMin) * (this.counts.length() - 3) / (this.binsMax - this.binsMin)); + this.counts.addAndGet(pos, count); + } + + // hasData needs to be updated as last + this.hasData.set(true); + } + + /** + * Computes the quantiles give the ratios. + */ + public long[] getQuantiles(double[] quantiles) { + long[] res = new long[quantiles.length]; + getQuantiles(quantiles, res); + return res; + } + + /** + * Computes the quantiles give the ratios. + */ + public void getQuantiles(double[] quantiles, long[] res) { + if (!this.hasData.get()) { + // No data yet. + for (int i = 0; i < res.length; i++) { + res[i] = 0L; + } + return; + } + + // Make a snapshot of lowerCounter, higherCounter and bins.counts to counts. + // This is not synchronized, but since the counter are accumulating, the result is a good + // estimation of a snapshot. + long[] counts = new long[this.counts.length()]; + long total = 0L; + for (int i = 0; i < this.counts.length(); i++) { + counts[i] = this.counts.get(i); + total += counts[i]; + } + + int rIndex = 0; + double qCount = total * quantiles[0]; + long cum = 0L; + + countsLoop: for (int i = 0; i < counts.length; i++) { + // mn and mx define a value range + long mn, mx; + if (i == 0) { + mn = this.min.get(); + mx = this.binsMin; + } else if (i == counts.length - 1) { + mn = this.bins10XMax; + mx = this.max.get(); + } else if (i == counts.length - 2) { + mn = this.binsMax; + mx = this.bins10XMax; + } else { + mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length() - 3); + mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length() - 3); + } + + if (mx < this.min.get()) { + continue; + } + if (mn > this.max.get()) { + break; + } + mn = Math.max(mn, this.min.get()); + mx = Math.min(mx, this.max.get()); + + // lastCum/cum are the corresponding counts to mn/mx + double lastCum = cum; + cum += counts[i]; + + // fill the results for qCount is within current range. + while (qCount <= cum) { + if (cum == lastCum) { + res[rIndex] = mn; + } else { + res[rIndex] = (long) ((qCount - lastCum) * (mx - mn) / (cum - lastCum) + mn); + } + + // move to next quantile + rIndex++; + if (rIndex >= quantiles.length) { + break countsLoop; + } + qCount = total * quantiles[rIndex]; + } + } + // In case quantiles contains values >= 100% + for (; rIndex < quantiles.length; rIndex++) { + res[rIndex] = this.max.get(); + } + + return; + } + + /** + * Returns the total number of points collected. + */ + public long getTotal() { + long total = 0L; + for (int i = 0; i < counts.length(); i++) { + total += counts.get(i); + } + return total; + } + + public long getMin() { + if (!this.hasData.get()) { + return 0L; + } + + return this.min.get(); + } + + public long getMax() { + if (!this.hasData.get()) { + return 0L; + } + + return this.max.get(); + } + } + + // The bins counting values. It is replaced with a new one in calling of reset(). + private volatile Bins bins = new Bins(); + // The quantiles for creating a Bins with last Bins. + private final int numOfBins; + + /** + * Constructor. + * @param numOfBins the number of bins for the histogram. A larger value results in more precise + * results but with lower efficiency, and vice versus. + */ + public FastLongHistogram(int numOfBins) { + this.numOfBins = numOfBins; + } + + /** + * Constructor setting the bins assuming a uniform distribution within a range. + * @param numOfBins the number of bins for the histogram. A larger value results in more precise + * results but with lower efficiency, and vice versus. + * @param min lower bound of the region, inclusive. + * @param max higher bound of the region, inclusive. + */ + public FastLongHistogram(int numOfBins, long min, long max) { + this(numOfBins); + Bins bins = new Bins(); + bins.add(min, 1); + bins.add(max, 1); + this.bins = new Bins(bins, numOfBins, 0.01, 0.99); + } + + /** + * Adds a value to the histogram. + */ + public void add(long value, long count) { + this.bins.add(value, count); + } + + /** + * Computes the quantiles give the ratios. + * @param quantiles the quantiles, should be in ascending order + * @return an array of long with same length as quantiles. + */ + public long[] getQuantiles(double[] quantiles) { + return this.bins.getQuantiles(quantiles); + } + + /** + * Computes the quantiles give the ratios. + * @param quantiles the quantiles, should be in ascending order + * @param res will be filled with results, should contains spaces for at least the number of + * quantiles. + */ + public void getQuantiles(double[] quantiles, long[] res) { + this.bins.getQuantiles(quantiles, res); + } + + /** + * Resets the histogram for new counting. + */ + public void reset() { + if (this.bins.hasData.get()) { + this.bins = new Bins(this.bins, numOfBins, 0.01, 0.99); + } + } + + /** + * Returns the minimum value. + */ + public long getMin() { + return this.bins.getMin(); + } + + /** + * Returns the maximum value. + */ + public long getMax() { + return this.bins.getMax(); + } + + /** + * Returns the total number of points collected. + */ + public long getTotal() { + return this.bins.getTotal(); + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/Percentiles.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/Percentiles.java new file mode 100644 index 0000000..2e502e7 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/util/Percentiles.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.concurrent.atomic.AtomicLong; + +import javax.annotation.concurrent.ThreadSafe; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.hadoop.metrics2.lib.MutableMetric; + +/** + * Percentiles estimates some percentiles of the appended data. It is thread-safe but not + * synchronized. + */ +@ThreadSafe +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class Percentiles extends MutableMetric implements MetricHistogram { + private static final double[] QUANTILES = { 0.50, 0.75, 0.95, 0.99 }; + private static final int QINDEX_50 = 0; + private static final int QINDEX_75 = 1; + private static final int QINDEX_95 = 2; + private static final int QINDEX_99 = 3; + + /** + * A data structure storing a set of metrics. + */ + private static class Metrics { + public final long[] metrics = new long[QUANTILES.length]; + public long min; + public long max; + public long sum; + public long count; + } + + private final String name; + private final String desc; + + private final FastLongHistogram hist; + private final AtomicLong sum = new AtomicLong(0L); + // The timestamp for next update/reset. + private volatile long nextResetTS = 0L; + // Current computed metrics. + private volatile Metrics currentMetrics = new Metrics(); + // A temporary buffer for Metrics. It is swapped with currentMetrics in every update. + private volatile Metrics bufferMetrics = new Metrics(); + // refresh period in milliseconds. + private final long msRefreshPeriod; + + /** + * Constructor. + * @param msRefreshPeriod refresh period in milliseconds. + */ + public Percentiles(String name, String description, long msRefreshPeriod) { + this.name = StringUtils.capitalize(name); + this.desc = StringUtils.uncapitalize(description); + + this.msRefreshPeriod = msRefreshPeriod; + this.hist = new FastLongHistogram(1024, 0, 10000); + this.nextResetTS = System.currentTimeMillis() + msRefreshPeriod; + } + + /** + * Adds a value to the histogram. + */ + @Override + public void add(long value) { + hist.add(value, 1); + sum.addAndGet(value); + } + + /** + * Updates the currentMetrics if necessary. + */ + private void updateMetrics() { + long now = System.currentTimeMillis(); + if (now < this.nextResetTS) { + return; + } + + synchronized (this) { + if (now < this.nextResetTS) { + return; + } + + // Computes the Metrics. + hist.getQuantiles(QUANTILES, bufferMetrics.metrics); + bufferMetrics.min = hist.getMin(); + bufferMetrics.max = hist.getMax(); + bufferMetrics.count = hist.getTotal(); + bufferMetrics.sum = sum.get(); + + sum.set(0L); + hist.reset(); + + // now swap bufferMetrics and cachedMetrics + Metrics tmp = currentMetrics; + currentMetrics = bufferMetrics; + bufferMetrics = tmp; + + // Updates nextResetTS to next update time. + this.nextResetTS = now + msRefreshPeriod; + } + } + + /** + * Used for testcase only. + */ + long getMin() { + updateMetrics(); + return currentMetrics.min; + } + + /** + * Used for testcase only. + */ + long getMax() { + updateMetrics(); + return currentMetrics.max; + } + + /** + * Used for testcase only. + */ + long getCount() { + updateMetrics(); + return currentMetrics.count; + } + + /** + * Used for testcase only. + */ + long[] getMetrics() { + updateMetrics(); + return currentMetrics.metrics; + } + + /** + * Used internally and for testcase only. + */ + long getMean() { + updateMetrics(); + + long count = currentMetrics.count; + if (count == 0) { + return 0L; + } + + return currentMetrics.sum / count; + } + + @Override + public void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { + updateMetrics(); + long mean = getMean(); + + metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), + currentMetrics.count); + + metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), currentMetrics.min); + metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), currentMetrics.max); + metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), mean); + + metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), + currentMetrics.metrics[QINDEX_50]); + metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + currentMetrics.metrics[QINDEX_75]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc), + currentMetrics.metrics[QINDEX_95]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc), + currentMetrics.metrics[QINDEX_99]); + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java index 281200c..ce5aaf5 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.Percentiles; import org.apache.hadoop.metrics2.MetricsException; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; @@ -286,6 +287,11 @@ public class DynamicMetricsRegistry { return addNewMetricIfAbsent(name, histo, MetricMutableQuantiles.class); } + public Percentiles newPercentiles(String name, String desc) { + Percentiles p = new Percentiles(name, desc, 60 * 1000L); + return addNewMetricIfAbsent(name, p, Percentiles.class); + } + synchronized void add(String name, MutableMetric metric) { addNewMetricIfAbsent(name, metric, MutableMetric.class); } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestAtomicUtils.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestAtomicUtils.java new file mode 100644 index 0000000..5a5a832 --- /dev/null +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestAtomicUtils.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Test functions in AtomicUtils. + */ +public class TestAtomicUtils { + + @Test + public void testUpdateMinMax() { + AtomicLong min = new AtomicLong(Long.MAX_VALUE); + AtomicLong max = new AtomicLong(Long.MIN_VALUE); + long realMin = Long.MAX_VALUE; + long realMax = Long.MIN_VALUE; + Random rand = new Random(); + for (int i = 0; i < 1000; i++) { + long value = rand.nextLong(); + + AtomicUtils.updateMin(min, value); + realMin = Math.min(realMin, value); + + AtomicUtils.updateMax(max, value); + realMax = Math.max(realMax, value); + } + + Assert.assertEquals("min", realMin, min.get()); + Assert.assertEquals("max", realMax, max.get()); + } + +} diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java new file mode 100644 index 0000000..f5848f3 --- /dev/null +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.Arrays; +import java.util.Random; + +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Testcases for FastLongHistogram. + */ +@Category({MiscTests.class, SmallTests.class}) +public class TestFastLongHistogram { + + private static void doTestUniform(FastLongHistogram hist) { + long[] VALUES = { 0, 10, 20, 30, 40, 50 }; + double[] qs = new double[VALUES.length]; + for (int i = 0; i < qs.length; i++) { + qs[i] = (double) VALUES[i] / VALUES[VALUES.length - 1]; + } + + for (int i = 0; i < 10; i++) { + for (long v : VALUES) { + hist.add(v, 1); + } + long[] vals = hist.getQuantiles(qs); + System.out.println(Arrays.toString(vals)); + for (int j = 0; j < qs.length; j++) { + Assert.assertTrue(j + "-th element org: " + VALUES[j] + ", act: " + vals[j], + Math.abs(vals[j] - VALUES[j]) <= 10); + } + hist.reset(); + } + } + + @Test + public void testUniform() { + FastLongHistogram hist = new FastLongHistogram(100, 0, 50); + doTestUniform(hist); + } + + @Test + public void testAdaptionOfChange() { + // assumes the uniform distribution + FastLongHistogram hist = new FastLongHistogram(100, 0, 100); + + Random rand = new Random(); + + for (int n = 0; n < 10; n++) { + for (int i = 0; i < 900; i++) { + hist.add(rand.nextInt(100), 1); + } + + // add 10% outliers, this breaks the assumption, hope bin10xMax works + for (int i = 0; i < 100; i++) { + hist.add(1000 + rand.nextInt(100), 1); + } + + long[] vals = hist.getQuantiles(new double[] { 0.25, 0.75, 0.95 }); + System.out.println(Arrays.toString(vals)); + if (n == 0) { + Assert.assertTrue("Out of possible value", vals[0] >= 0 && vals[0] <= 50); + Assert.assertTrue("Out of possible value", vals[1] >= 50 && vals[1] <= 100); + Assert.assertTrue("Out of possible value", vals[2] >= 900 && vals[2] <= 1100); + } + + hist.reset(); + } + } + + @Test + public void testSameValues() { + FastLongHistogram hist = new FastLongHistogram(100); + + hist.add(50, 100); + + hist.reset(); + doTestUniform(hist); + } +} diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestPercentiles.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestPercentiles.java new file mode 100644 index 0000000..7b901d1 --- /dev/null +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/util/TestPercentiles.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.Arrays; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Testcases for Percentiles. + */ +public class TestPercentiles { + + private static void assertArrayApproximate(String name, long[] exp, long[] act, long err) { + for (int i = 0; i < exp.length; i++) { + if (Math.abs(exp[i] - act[i]) > err) { + Assert.fail(String.format( + "The value of the %d-th element of %s is %d which is far away from expected value %d", + i, name, act[i], exp[i])); + } + } + } + + @Test + public void testBasic() throws Exception { + Percentiles p = new Percentiles("", "", 100L); + // Should return zeros and more importantly throw no exceptions before feeding any data. + Assert.assertEquals("count", 0, p.getCount()); + Assert.assertEquals("min", 0, p.getMin()); + Assert.assertEquals("max", 0, p.getMax()); + Assert.assertArrayEquals("metrics", new long[] { 0, 0, 0, 0 }, p.getMetrics()); + + // Go over the update time. + Thread.sleep(150L); + + // Should return zeros and more importantly throw no exceptions before feeding any data. + Assert.assertEquals("count", 0, p.getCount()); + Assert.assertEquals("min", 0, p.getMin()); + Assert.assertEquals("max", 0, p.getMax()); + Assert.assertArrayEquals("metrics", new long[] { 0, 0, 0, 0 }, p.getMetrics()); + + // Feed 100 points linearly + for (int i = 0; i <= 100; i++) { + p.add(i*10); + } + + // Go over the update time. + Thread.sleep(150L); + + // Check metrics + Assert.assertEquals("count", 101, p.getCount()); + Assert.assertEquals("min", 0, p.getMin()); + Assert.assertEquals("max", 1000, p.getMax()); + System.out.println("metrics: " + Arrays.toString(p.getMetrics())); + assertArrayApproximate("metrics", new long[] { 500, 750, 950, 990 }, p.getMetrics(), 10); + } + +} -- 1.9.3 (Apple Git-50)