diff --git hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLogSource.java hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLogSource.java new file mode 100644 index 0000000..fd723dc --- /dev/null +++ hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLogSource.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.hbase.metrics.BaseSource; + +/** + * Interface of the source that will export metrics about the region server's HLog. + */ +public interface MetricsHLogSource extends BaseSource { + + + /** + * The name of the metrics + */ + static final String METRICS_NAME = "HLog"; + + /** + * The name of the metrics context that metrics will be under. + */ + static final String METRICS_CONTEXT = "regionserver"; + + /** + * Description + */ + static final String METRICS_DESCRIPTION = "Metrics about HBase RegionServer HLog"; + + /** + * The name of the metrics context that metrics will be under in jmx + */ + static final String METRICS_JMX_CONTEXT = "RegionServer,sub=" + METRICS_NAME; + + + static final String APPEND_TIME = "appendTime"; + static final String APPEND_TIME_DESC = "Time an append to the log took"; + static final String APPEND_COUNT = "appendCount"; + static final String APPEND_COUNT_DESC = "Number of appends to the write ahead log."; + static final String APPEND_SIZE = "appendSize"; + static final String APPEND_SIZE_DESC = "Size of the data appended to the HLog"; + static final String SLOW_APPEND_COUNT = "slowAppendCount"; + static final String SLOW_APPEND_COUNT_DESC = "Number of appends that were slow."; + static final String SYNC_TIME = "syncTime"; + static final String SYNC_TIME_DESC = "The time it took to sync the HLog to HDFS."; + + /** + * Add the append size to a histogram. + */ + void incrementAppendSize(long size); + + /** + * Add the time it to to append to a histogram. + */ + void incrementAppendTime(long time); + + /** + * Increment the count of hlog appends + */ + void incrementAppendCount(); + + /** + * Increment the number of appends that were slow + */ + void incrementSlowAppendCount(); + + /** + * Add the time it took to sync the hlog to a histogram. + */ + void incrementSyncTime(long time); + +} diff --git hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsHLogSource.java hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsHLogSource.java new file mode 100644 index 0000000..750bee4 --- /dev/null +++ hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsHLogSource.java @@ -0,0 +1,14 @@ +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.junit.Test; + +public class TestMetricsHLogSource { + + @Test(expected=RuntimeException.class) + public void testGetInstanceNoHadoopCompat() throws Exception { + //This should throw an exception because there is no compat lib on the class path. + CompatibilitySingletonFactory.getInstance(MetricsHLogSource.class); + + } +} diff --git hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLogSourceImpl.java hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLogSourceImpl.java new file mode 100644 index 0000000..2136773 --- /dev/null +++ hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLogSourceImpl.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.lib.MetricMutableCounterLong; + + +public class MetricsHLogSourceImpl extends BaseSourceImpl implements MetricsHLogSource{ + + private final MetricHistogram appendSizeHisto; + private final MetricHistogram appendTimeHisto; + private final MetricMutableCounterLong appendCount; + private final MetricMutableCounterLong slowAppendCount; + private final MetricHistogram syncTimeHisto; + + + public MetricsHLogSourceImpl() { + this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); + } + + public MetricsHLogSourceImpl(String metricsName, + String metricsDescription, + String metricsContext, + String metricsJmxContext) { + super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + + //Create and store the metrics that will be used. + appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC); + appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC); + appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l); + slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l); + syncTimeHisto = this.getMetricsRegistry().newHistogram(SYNC_TIME, SYNC_TIME_DESC); + } + + @Override + public void incrementAppendSize(long size) { + appendSizeHisto.add(size); + } + + @Override + public void incrementAppendTime(long time) { + appendTimeHisto.add(time); + } + + @Override + public void incrementAppendCount() { + appendCount.incr(); + } + + @Override + public void incrementSlowAppendCount() { + slowAppendCount.incr(); + } + + @Override + public void incrementSyncTime(long time) { + syncTimeHisto.add(time); + } +} diff --git hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.wal.MetricsHLogSource hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.wal.MetricsHLogSource new file mode 100644 index 0000000..e82eaed --- /dev/null +++ hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.wal.MetricsHLogSource @@ -0,0 +1 @@ +org.apache.hadoop.hbase.regionserver.wal.MetricsHLogSourceImpl diff --git hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsHLogSourceImpl.java hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsHLogSourceImpl.java new file mode 100644 index 0000000..665fc36 --- /dev/null +++ hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsHLogSourceImpl.java @@ -0,0 +1,20 @@ +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceImpl; +import org.junit.Test; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class TestMetricsHLogSourceImpl { + + @Test + public void testGetInstance() throws Exception { + MetricsHLogSource hLogSource = + CompatibilitySingletonFactory.getInstance(MetricsHLogSource.class); + assertTrue(hLogSource instanceof MetricsHLogSourceImpl); + assertSame(hLogSource, + CompatibilitySingletonFactory.getInstance(MetricsHLogSource.class)); + } +} diff --git hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLogSourceImpl.java hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLogSourceImpl.java new file mode 100644 index 0000000..955fe8c --- /dev/null +++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLogSourceImpl.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; + + +public class MetricsHLogSourceImpl extends BaseSourceImpl implements MetricsHLogSource { + + private final MetricHistogram appendSizeHisto; + private final MetricHistogram appendTimeHisto; + private final MetricHistogram syncTimeHisto; + private final MutableCounterLong appendCount; + private final MutableCounterLong slowAppendCount; + + public MetricsHLogSourceImpl() { + this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); + } + + public MetricsHLogSourceImpl(String metricsName, + String metricsDescription, + String metricsContext, + String metricsJmxContext) { + super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + + //Create and store the metrics that will be used. + appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC); + appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC); + appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l); + slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l); + syncTimeHisto = this.getMetricsRegistry().newHistogram(SYNC_TIME, SYNC_TIME_DESC); + } + + @Override + public void incrementAppendSize(long size) { + appendSizeHisto.add(size); + } + + @Override + public void incrementAppendTime(long time) { + appendTimeHisto.add(time); + } + + @Override + public void incrementAppendCount() { + appendCount.incr(); + } + + @Override + public void incrementSlowAppendCount() { + slowAppendCount.incr(); + } + + @Override + public void incrementSyncTime(long time) { + syncTimeHisto.add(time); + } +} diff --git hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.wal.MetricsHLogSource hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.wal.MetricsHLogSource new file mode 100644 index 0000000..e82eaed --- /dev/null +++ hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.wal.MetricsHLogSource @@ -0,0 +1 @@ +org.apache.hadoop.hbase.regionserver.wal.MetricsHLogSourceImpl diff --git hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsHLogSourceImpl.java hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsHLogSourceImpl.java new file mode 100644 index 0000000..665fc36 --- /dev/null +++ hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsHLogSourceImpl.java @@ -0,0 +1,20 @@ +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceImpl; +import org.junit.Test; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class TestMetricsHLogSourceImpl { + + @Test + public void testGetInstance() throws Exception { + MetricsHLogSource hLogSource = + CompatibilitySingletonFactory.getInstance(MetricsHLogSource.class); + assertTrue(hLogSource instanceof MetricsHLogSourceImpl); + assertSame(hLogSource, + CompatibilitySingletonFactory.getInstance(MetricsHLogSource.class)); + } +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index 8645c8b..d9373ac 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -18,22 +18,17 @@ */ package org.apache.hadoop.hbase.regionserver.wal; -import java.io.DataInput; -import java.io.DataOutput; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URLEncoder; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.NavigableSet; import java.util.SortedMap; import java.util.TreeMap; import java.util.TreeSet; @@ -45,8 +40,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -56,16 +49,12 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.Syncable; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader; -import org.apache.hadoop.hbase.regionserver.wal.HLog.Writer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.FSUtils; @@ -214,7 +203,8 @@ class FSHLog implements HLog, Syncable { private final int closeErrorsTolerated; private final AtomicInteger closeErrorCount = new AtomicInteger(); - + private final MetricsHLog metrics; + /** * Constructor. * @@ -365,6 +355,8 @@ class FSHLog implements HLog, Syncable { Threads.setDaemonThreadRunning(logSyncerThread.getThread(), Thread.currentThread().getName() + ".logSyncer"); coprocessorHost = new WALCoprocessorHost(this, conf); + + this.metrics = new MetricsHLog(); } // use reflection to search for getDefaultBlockSize(Path f) @@ -1081,7 +1073,7 @@ class FSHLog implements HLog, Syncable { } this.syncedTillHere = Math.max(this.syncedTillHere, doneUpto); - HLogMetrics.syncTime.inc(System.currentTimeMillis() - now); + this.metrics.finishSync(System.currentTimeMillis() - now); if (!this.logRollRunning) { checkLowReplication(); try { @@ -1216,20 +1208,11 @@ class FSHLog implements HLog, Syncable { } long took = System.currentTimeMillis() - now; coprocessorHost.postWALWrite(info, logKey, logEdit); - HLogMetrics.writeTime.inc(took); long len = 0; for (KeyValue kv : logEdit.getKeyValues()) { len += kv.getLength(); } - HLogMetrics.writeSize.inc(len); - if (took > 1000) { - LOG.warn(String.format( - "%s took %d ms appending an edit to hlog; editcount=%d, len~=%s", - Thread.currentThread().getName(), took, this.numEntries.get(), - StringUtils.humanReadableInt(len))); - HLogMetrics.slowHLogAppendCount.incrementAndGet(); - HLogMetrics.slowHLogAppendTime.inc(took); - } + this.metrics.finishAppend(took, len); } catch (IOException e) { LOG.fatal("Could not append. Requesting close of hlog", e); requestLogRoll(); @@ -1305,12 +1288,12 @@ class FSHLog implements HLog, Syncable { System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID); logSyncerThread.append(new Entry(key, edit)); txid = this.unflushedEntries.incrementAndGet(); - HLogMetrics.writeTime.inc(System.currentTimeMillis() - now); + long took = System.currentTimeMillis() - now; long len = 0; for (KeyValue kv : edit.getKeyValues()) { len += kv.getLength(); } - HLogMetrics.writeSize.inc(len); + this.metrics.finishAppend(took, len); this.numEntries.incrementAndGet(); } // sync txn to file system diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java index b39f8ea..763b8d4 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java @@ -36,7 +36,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Writable; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.regionserver.wal.HLogMetrics.Metric; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.classification.InterfaceAudience; diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogMetrics.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogMetrics.java deleted file mode 100644 index 62f8e83..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogMetrics.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver.wal; - -import java.util.concurrent.atomic.AtomicLong; - - -public class HLogMetrics { - - public static class Metric { - public long min = Long.MAX_VALUE; - public long max = 0; - public long total = 0; - public int count = 0; - - synchronized void inc(final long val) { - min = Math.min(min, val); - max = Math.max(max, val); - total += val; - ++count; - } - - synchronized Metric get() { - Metric copy = new Metric(); - copy.min = min; - copy.max = max; - copy.total = total; - copy.count = count; - this.min = Long.MAX_VALUE; - this.max = 0; - this.total = 0; - this.count = 0; - return copy; - } - } - - - // For measuring latency of writes - static Metric writeTime = new Metric(); - static Metric writeSize = new Metric(); - // For measuring latency of syncs - static Metric syncTime = new Metric(); - //For measuring slow HLog appends - static AtomicLong slowHLogAppendCount = new AtomicLong(); - static Metric slowHLogAppendTime = new Metric(); - - public static Metric getWriteTime() { - return writeTime.get(); - } - - public static Metric getWriteSize() { - return writeSize.get(); - } - - public static Metric getSyncTime() { - return syncTime.get(); - } - - public static long getSlowAppendCount() { - return slowHLogAppendCount.get(); - } - - public static Metric getSlowAppendTime() { - return slowHLogAppendTime.get(); - } -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLog.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLog.java new file mode 100644 index 0000000..65a5a30 --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsHLog.java @@ -0,0 +1,57 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.util.StringUtils; + +@InterfaceAudience.Private +public class MetricsHLog { + static final Log LOG = LogFactory.getLog(MetricsHLog.class); + + private MetricsHLogSource source; + + public MetricsHLog() { + source = CompatibilitySingletonFactory.getInstance(MetricsHLogSource.class); + } + + public void finishSync(long time) { + source.incrementSyncTime(time); + + } + + public void finishAppend(long time, long size) { + + source.incrementAppendCount(); + source.incrementAppendTime(time); + source.incrementAppendSize(size); + + if (time > 1000) { + source.incrementSlowAppendCount(); + LOG.warn(String.format("%s took %d ms appending an edit to hlog; len~=%s", + Thread.currentThread().getName(), + time, + StringUtils.humanReadableInt(size))); + } + } +} diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index d5a3149..0f2001f 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -36,6 +36,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.FailedSanityCheckException; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -56,13 +57,11 @@ import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; @@ -72,7 +71,6 @@ import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.NullComparator; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; -import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; @@ -82,13 +80,13 @@ import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLogFactory; import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.regionserver.wal.HLogMetrics; +import org.apache.hadoop.hbase.regionserver.wal.MetricsHLog; +import org.apache.hadoop.hbase.regionserver.wal.MetricsHLogSource; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; -import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.Threads; @@ -130,6 +128,9 @@ public class TestHRegion extends HBaseTestCase { protected final byte [] row = Bytes.toBytes("rowA"); protected final byte [] row2 = Bytes.toBytes("rowB"); + protected final MetricsAssertHelper metricsAssertHelper = + CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class); + /** * @see org.apache.hadoop.hbase.HBaseTestCase#setUp() @@ -628,9 +629,10 @@ public class TestHRegion extends HBaseTestCase { byte[] qual = Bytes.toBytes("qual"); byte[] val = Bytes.toBytes("val"); this.region = initHRegion(b, getName(), cf); + MetricsHLogSource source = CompatibilitySingletonFactory.getInstance(MetricsHLogSource.class); try { - HLogMetrics.getSyncTime(); // clear counter from prior tests - assertEquals(0, HLogMetrics.getSyncTime().count); + long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); LOG.info("First a batch put with all valid puts"); final Put[] puts = new Put[10]; @@ -645,7 +647,8 @@ public class TestHRegion extends HBaseTestCase { assertEquals(OperationStatusCode.SUCCESS, codes[i] .getOperationStatusCode()); } - assertEquals(1, HLogMetrics.getSyncTime().count); + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); + LOG.info("Next a batch put with one invalid family"); puts[5].add(Bytes.toBytes("BAD_CF"), qual, val); @@ -655,7 +658,8 @@ public class TestHRegion extends HBaseTestCase { assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); } - assertEquals(1, HLogMetrics.getSyncTime().count); + + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source); LOG.info("Next a batch put that has to break into two batches to avoid a lock"); Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2")); @@ -676,7 +680,7 @@ public class TestHRegion extends HBaseTestCase { LOG.info("...waiting for put thread to sync first time"); long startWait = System.currentTimeMillis(); - while (HLogMetrics.getSyncTime().count == 0) { + while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs +2 ) { Thread.sleep(100); if (System.currentTimeMillis() - startWait > 10000) { fail("Timed out waiting for thread to sync first minibatch"); @@ -687,7 +691,7 @@ public class TestHRegion extends HBaseTestCase { LOG.info("...joining on thread"); ctx.stop(); LOG.info("...checking that next batch was synced"); - assertEquals(1, HLogMetrics.getSyncTime().count); + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source); codes = retFromThread.get(); for (int i = 0; i < 10; i++) { assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : @@ -711,7 +715,7 @@ public class TestHRegion extends HBaseTestCase { OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); } // Make sure we didn't do an extra batch - assertEquals(1, HLogMetrics.getSyncTime().count); + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 5, source); // Make sure we still hold lock assertTrue(region.isRowLocked(lockedRow)); @@ -737,8 +741,9 @@ public class TestHRegion extends HBaseTestCase { this.region = initHRegion(b, getName(), conf, cf); try{ - HLogMetrics.getSyncTime(); // clear counter from prior tests - assertEquals(0, HLogMetrics.getSyncTime().count); + MetricsHLogSource source = CompatibilitySingletonFactory.getInstance(MetricsHLogSource.class); + long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); final Put[] puts = new Put[10]; for (int i = 0; i < 10; i++) { @@ -752,8 +757,7 @@ public class TestHRegion extends HBaseTestCase { assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i] .getOperationStatusCode()); } - assertEquals(0, HLogMetrics.getSyncTime().count); - + metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); } finally { HRegion.closeHRegion(this.region);