diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 71e1d28..43ad01c 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -73,7 +73,8 @@ public class MetricsRegionServerSourceImpl deleteHisto = getMetricsRegistry().newHistogram(DELETE_KEY); slowDelete = getMetricsRegistry().newCounter(SLOW_DELETE_KEY, SLOW_DELETE_DESC, 0l); - getHisto = getMetricsRegistry().newHistogram(GET_KEY); + //getHisto = getMetricsRegistry().newHistogram(GET_KEY); + getHisto = getMetricsRegistry().newTimeHistogram(GET_KEY); slowGet = getMetricsRegistry().newCounter(SLOW_GET_KEY, SLOW_GET_DESC, 0l); incrementHisto = getMetricsRegistry().newHistogram(INCREMENT_KEY); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java index f6f12b8..d25bf5a 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java @@ -51,8 +51,10 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo super(metricsName, metricsDescription, metricsContext, metricsJmxContext); //Create and store the metrics that will be used. - appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC); - appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC); + //appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC); + appendTimeHisto = this.getMetricsRegistry().newTimeHistogram(APPEND_TIME, APPEND_TIME_DESC); + //appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC); + appendSizeHisto = this.getMetricsRegistry().newSizeHistogram(APPEND_SIZE, APPEND_SIZE_DESC); appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l); slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java index 281200c..daa597b 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java @@ -273,6 +273,46 @@ public class DynamicMetricsRegistry { } /** + * Create a new histogram with time range counters. + * @param name Name of the histogram. + * @return A new MutableTimeHistogram + */ + public MutableTimeHistogram newTimeHistogram(String name) { + return newTimeHistogram(name, ""); + } + + /** + * Create a new histogram with time range counters. + * @param name The name of the histogram + * @param desc The description of the data in the histogram. + * @return A new MutableTimeHistogram + */ + public MutableTimeHistogram newTimeHistogram(String name, String desc) { + MutableTimeHistogram histo = new MutableTimeHistogram(name, desc); + return addNewMetricIfAbsent(name, histo, MutableTimeHistogram.class); + } + + /** + * Create a new histogram with size range counters. + * @param name Name of the histogram. + * @return A new MutableSizeHistogram + */ + public MutableSizeHistogram newSizeHistogram(String name) { + return newSizeHistogram(name, ""); + } + + /** + * Create a new histogram with size range counters. + * @param name The name of the histogram + * @param desc The description of the data in the histogram. + * @return A new MutableSizeHistogram + */ + public MutableSizeHistogram newSizeHistogram(String name, String desc) { + MutableSizeHistogram histo = new MutableSizeHistogram(name, desc); + return addNewMetricIfAbsent(name, histo, MutableSizeHistogram.class); + } + + /** * Create a new MutableQuantile(A more accurate histogram). * @param name The name of the histogram * @return a new MutableQuantile diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java index 381dbbb..1940e4c 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java @@ -18,8 +18,6 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricHistogram; @@ -44,10 +42,7 @@ public class MutableHistogram extends MutableMetric implements MetricHistogram { private final String name; private final String desc; private final Sample sample; - private final AtomicLong min; - private final AtomicLong max; - private final AtomicLong sum; - private final AtomicLong count; + private final MutableStats mstats; public MutableHistogram(MetricsInfo info) { this(info.name(), info.description()); @@ -57,59 +52,13 @@ public class MutableHistogram extends MutableMetric implements MetricHistogram { this.name = StringUtils.capitalize(name); this.desc = StringUtils.uncapitalize(description); sample = new ExponentiallyDecayingSample(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA); - count = new AtomicLong(); - min = new AtomicLong(Long.MAX_VALUE); - max = new AtomicLong(Long.MIN_VALUE); - sum = new AtomicLong(); + mstats = new MutableStats(); } public void add(final long val) { setChanged(); - count.incrementAndGet(); + mstats.add(val); sample.update(val); - setMax(val); - setMin(val); - sum.getAndAdd(val); - } - - private void setMax(final long potentialMax) { - boolean done = false; - while (!done) { - final long currentMax = max.get(); - done = currentMax >= potentialMax - || max.compareAndSet(currentMax, potentialMax); - } - } - - private void setMin(long potentialMin) { - boolean done = false; - while (!done) { - final long currentMin = min.get(); - done = currentMin <= potentialMin - || min.compareAndSet(currentMin, potentialMin); - } - } - - public long getMax() { - if (count.get() > 0) { - return max.get(); - } - return 0L; - } - - public long getMin() { - if (count.get() > 0) { - return min.get(); - } - return 0L; - } - - public double getMean() { - long cCount = count.get(); - if (cCount > 0) { - return sum.get() / (double) cCount; - } - return 0.0; } @Override @@ -117,11 +66,11 @@ public class MutableHistogram extends MutableMetric implements MetricHistogram { if (all || changed()) { clearChanged(); final Snapshot s = sample.getSnapshot(); - metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), count.get()); + metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), mstats.getCount()); - metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), getMin()); - metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), getMax()); - metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), getMean()); + metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), mstats.getMin()); + metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), mstats.getMax()); + metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), mstats.getMean()); metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), s.getMedian()); metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java index e69de29..9e52eeb 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.util.concurrent.atomic.AtomicLongArray; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +import com.yammer.metrics.stats.ExponentiallyDecayingSample; +import com.yammer.metrics.stats.Sample; +import com.yammer.metrics.stats.Snapshot; + +/** + * A histogram implementation that runs in constant space, and exports to hadoop2's metrics2 system. + */ +@InterfaceAudience.Private +public class MutableSizeHistogram extends MutableMetric implements MetricHistogram { + + private static final int DEFAULT_SAMPLE_SIZE = 2046; + // the bias towards sampling from more recent data. + // Per Cormode et al. an alpha of 0.015 strongly biases to the last 5 minutes + private static final double DEFAULT_ALPHA = 0.015; + + private final String name; + private final String desc; + private final Sample sample; + private final MutableStats mstats; + private final long[] ranges = {10,100,1000,10000,100000,1000000,10000000,100000000}; + private final AtomicLongArray rangeVals = new AtomicLongArray(ranges.length+1); + + public MutableSizeHistogram(MetricsInfo info) { + this(info.name(), info.description()); + } + + public MutableSizeHistogram(String name, String description) { + this.name = StringUtils.capitalize(name); + this.desc = StringUtils.uncapitalize(description); + mstats = new MutableStats(); + sample = new ExponentiallyDecayingSample(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA); + } + + public void add(final long val) { + setChanged(); + sample.update(val); + mstats.add(val); + updateBand(val); + } + + private void updateBand(final long val) { + int i; + for (i=0; i ranges[i]; i++); + rangeVals.incrementAndGet(i); + } + + @Override + public void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { + if (all || changed()) { + clearChanged(); + final Snapshot s = sample.getSnapshot(); + metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), mstats.getCount()); + + metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), mstats.getMin()); + metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), mstats.getMax()); + metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), mstats.getMean()); + + metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), s.getMedian()); + metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + s.get75thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc), + s.get95thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc), + s.get99thPercentile()); + + long prior = 0; + for (int i = 0; i < ranges.length; i++) { + long val = rangeVals.get(i); + if (val > 0) { + metricsRecordBuilder + .addCounter(Interns.info(name + "_SizeCount_" + prior + "-" + ranges[i], desc), val); + } + prior = ranges[i]; + } + long val = rangeVals.get(ranges.length); + if (val > 0) { + metricsRecordBuilder.addCounter( + Interns.info(name + "_>" + ranges[ranges.length - 1], desc), + rangeVals.get(ranges.length)); + } + } + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableStats.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableStats.java index e69de29..549894a 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableStats.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableStats.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * Basic min-mean-max stats implementation for hadoop2's metrics2 system. + */ +@InterfaceAudience.Private +public class MutableStats { + private final AtomicLong min; + private final AtomicLong max; + private final AtomicLong sum; + private final AtomicLong count; + + public MutableStats() { + count = new AtomicLong(); + min = new AtomicLong(Long.MAX_VALUE); + max = new AtomicLong(Long.MIN_VALUE); + sum = new AtomicLong(); + } + + public void add(final long val) { + count.incrementAndGet(); + setMax(val); + setMin(val); + sum.getAndAdd(val); + } + + private void setMax(final long potentialMax) { + boolean done = false; + while (!done) { + final long currentMax = max.get(); + done = currentMax >= potentialMax + || max.compareAndSet(currentMax, potentialMax); + } + } + + private void setMin(long potentialMin) { + boolean done = false; + while (!done) { + final long currentMin = min.get(); + done = currentMin <= potentialMin + || min.compareAndSet(currentMin, potentialMin); + } + } + + public long getMax() { + if (count.get() > 0) { + return max.get(); + } + return 0L; + } + + public long getMin() { + if (count.get() > 0) { + return min.get(); + } + return 0L; + } + + public double getMean() { + long cCount = count.get(); + if (cCount > 0) { + return sum.get() / (double) cCount; + } + return 0.0; + } + + public long getCount() { + return count.get(); + } + +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java index e69de29..bae08da 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.util.concurrent.atomic.AtomicLongArray; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +import com.yammer.metrics.stats.ExponentiallyDecayingSample; +import com.yammer.metrics.stats.Sample; +import com.yammer.metrics.stats.Snapshot; + +/** + * A histogram implementation that runs in constant space, and exports to hadoop2's metrics2 system. + */ +@InterfaceAudience.Private +public class MutableTimeHistogram extends MutableMetric implements MetricHistogram { + + private static final int DEFAULT_SAMPLE_SIZE = 2046; + // the bias towards sampling from more recent data. + // Per Cormode et al. an alpha of 0.015 strongly biases to the last 5 minutes + private static final double DEFAULT_ALPHA = 0.015; + + private final String name; + private final String desc; + private final Sample sample; + private final MutableStats mstats; + private final long[] ranges = {1,3,10,30,100,300,1000,3000,10000,30000,60000,120000,300000,600000}; + private final AtomicLongArray rangeVals = new AtomicLongArray(ranges.length+1); + + public MutableTimeHistogram(MetricsInfo info) { + this(info.name(), info.description()); + } + + public MutableTimeHistogram(String name, String description) { + this.name = StringUtils.capitalize(name); + this.desc = StringUtils.uncapitalize(description); + mstats = new MutableStats(); + sample = new ExponentiallyDecayingSample(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA); + } + + public void add(final long val) { + setChanged(); + mstats.add(val); + sample.update(val); + updateBand(val); + } + + private void updateBand(final long val) { + int i; + for (i=0; i ranges[i]; i++); + rangeVals.incrementAndGet(i); + } + + @Override + public void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { + if (all || changed()) { + clearChanged(); + final Snapshot s = sample.getSnapshot(); + metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), mstats.getCount()); + + metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), mstats.getMin()); + metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), mstats.getMax()); + metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), mstats.getMean()); + + metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), s.getMedian()); + metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + s.get75thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc), + s.get95thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc), + s.get99thPercentile()); + + long prior = 0; + for (int i = 0; i < ranges.length; i++) { + long val = rangeVals.get(i); + if (val > 0) { + metricsRecordBuilder + .addCounter(Interns.info(name + "_TimeCount_" + prior + "-" + ranges[i], desc), val); + } + prior = ranges[i]; + } + long val = rangeVals.get(ranges.length); + if (val > 0) { + metricsRecordBuilder.addCounter( + Interns.info(name + "_>" + ranges[ranges.length - 1], desc), + rangeVals.get(ranges.length)); + } + } + } +}