From 3334b2568cbcef65222527a23d162429b55ae2a3 Mon Sep 17 00:00:00 2001 From: Xu Cang Date: Fri, 27 Jul 2018 21:42:08 -0700 Subject: [PATCH] HBASE-20925 Canary test to expose table availability rate fix checkstyle --- .../java/org/apache/hadoop/hbase/tool/Canary.java | 259 ++++++++++++++++----- .../apache/hadoop/hbase/tool/TestCanaryTool.java | 16 +- 2 files changed, 210 insertions(+), 65 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java index ae6324f71b..05dadd2898 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java @@ -208,35 +208,43 @@ public final class Canary implements Tool { private Map perTableReadLatency = new HashMap<>(); private LongAdder writeLatency = new LongAdder(); + private Map perTableReadFailureCount = new HashMap<>(); + private Map perTableReadSuccessCount = new HashMap<>(); public void publishReadFailure(ServerName serverName, RegionInfo region, Exception e) { incReadFailureCount(); - LOG.error(String.format("read from region %s on regionserver %s failed", region.getRegionNameAsString(), serverName), e); + LOG.error(String.format("read from region %s on regionserver %s failed", + region.getRegionNameAsString(), serverName), e); } - public void publishReadFailure(ServerName serverName, RegionInfo region, ColumnFamilyDescriptor column, Exception e) { + public void publishReadFailure(ServerName serverName, RegionInfo region, + ColumnFamilyDescriptor column, Exception e) { incReadFailureCount(); LOG.error(String.format("read from region %s on regionserver %s column family %s failed", region.getRegionNameAsString(), serverName, column.getNameAsString()), e); } - public void publishReadTiming(ServerName serverName, RegionInfo region, ColumnFamilyDescriptor column, long msTime) { + public void publishReadTiming(ServerName serverName, RegionInfo region, + ColumnFamilyDescriptor column, long msTime) { LOG.info(String.format("read from region %s on regionserver %s column family %s in %dms", region.getRegionNameAsString(), serverName, column.getNameAsString(), msTime)); } public void publishWriteFailure(ServerName serverName, RegionInfo region, Exception e) { incWriteFailureCount(); - LOG.error(String.format("write to region %s on regionserver %s failed", region.getRegionNameAsString(), serverName), e); + LOG.error(String.format("write to region %s on regionserver %s failed", + region.getRegionNameAsString(), serverName), e); } - public void publishWriteFailure(ServerName serverName, RegionInfo region, ColumnFamilyDescriptor column, Exception e) { + public void publishWriteFailure(ServerName serverName, RegionInfo region, + ColumnFamilyDescriptor column, Exception e) { incWriteFailureCount(); LOG.error(String.format("write to region %s on regionserver %s column family %s failed", region.getRegionNameAsString(), serverName, column.getNameAsString()), e); } - public void publishWriteTiming(ServerName serverName, RegionInfo region, ColumnFamilyDescriptor column, long msTime) { + public void publishWriteTiming(ServerName serverName, RegionInfo region, + ColumnFamilyDescriptor column, long msTime) { LOG.info(String.format("write to region %s on regionserver %s column family %s in %dms", region.getRegionNameAsString(), serverName, column.getNameAsString(), msTime)); } @@ -251,6 +259,17 @@ public final class Canary implements Tool { return initLatency; } + public LongAdder initializeAndGetReadSuccessCountForTable(String tableName) { + LongAdder successCount = new LongAdder(); + this.perTableReadSuccessCount.put(tableName, successCount); + return successCount; + } + public LongAdder initializeAndGetReadFailureCountForTable(String tableName) { + LongAdder failureCount = new LongAdder(); + this.perTableReadFailureCount.put(tableName, failureCount); + return failureCount; + } + public void initializeWriteLatency() { this.writeLatency.reset(); } @@ -258,6 +277,15 @@ public final class Canary implements Tool { public LongAdder getWriteLatency() { return this.writeLatency; } + + public Map getPerTableReadSuccessCount() { + return perTableReadSuccessCount; + } + + public Map getPerTableReadFailureCount() { + return perTableReadFailureCount; + } + } static class ZookeeperTask implements Callable { @@ -313,8 +341,9 @@ public final class Canary implements Tool { private ServerName serverName; private LongAdder readWriteLatency; - RegionTask(Connection connection, RegionInfo region, ServerName serverName, RegionStdOutSink sink, - TaskType taskType, boolean rawScanEnabled, LongAdder rwLatency) { + RegionTask(Connection connection, RegionInfo region, ServerName serverName, + RegionStdOutSink sink, TaskType taskType, boolean rawScanEnabled, + LongAdder rwLatency) { this.connection = connection; this.region = region; this.serverName = serverName; @@ -404,9 +433,23 @@ public final class Canary implements Tool { stopWatch.stop(); this.readWriteLatency.add(stopWatch.getTime()); sink.publishReadTiming(serverName, region, column, stopWatch.getTime()); + if (sink.perTableReadSuccessCount.containsKey(region.getTable().getNameAsString())) { + sink.perTableReadSuccessCount.get(region.getTable().getNameAsString()).increment(); + } else { + LongAdder longAdder = new LongAdder(); + longAdder.increment(); + sink.perTableReadSuccessCount.put(region.getTable().getNameAsString(), longAdder); + } } catch (Exception e) { sink.publishReadFailure(serverName, region, column, e); sink.updateReadFailures(region.getRegionNameAsString(), serverName.getHostname()); + if (sink.perTableReadFailureCount.containsKey(region.getTable().getNameAsString())) { + sink.perTableReadFailureCount.get(region.getTable().getNameAsString()).increment(); + } else { + LongAdder longAdder = new LongAdder(); + longAdder.increment(); + sink.perTableReadFailureCount.put(region.getTable().getNameAsString(),longAdder); + } } finally { if (rs != null) { rs.close(); @@ -463,7 +506,7 @@ public final class Canary implements Tool { table.close(); } catch (IOException e) { sink.publishWriteFailure(serverName, region, e); - sink.updateWriteFailures(region.getRegionNameAsString(), serverName.getHostname() ); + sink.updateWriteFailures(region.getRegionNameAsString(), serverName.getHostname()); } return null; } @@ -586,10 +629,12 @@ public final class Canary implements Tool { private boolean failOnError = true; private boolean regionServerMode = false; private boolean zookeeperMode = false; + private boolean regionMode = false; private boolean regionServerAllRegions = false; private boolean writeSniffing = false; private long configuredWriteTableTimeout = DEFAULT_TIMEOUT; private boolean treatFailureAsError = false; + private boolean tableAvailabilitySummary = false; private TableName writeTableName = DEFAULT_WRITE_TABLE_NAME; private HashMap configuredReadTableTimeouts = new HashMap<>(); @@ -648,6 +693,8 @@ public final class Canary implements Tool { System.err.println("-interval needs a numeric value argument."); printUsageAndExit(); } + } else if (cmd.equals("-regionMode")) { + this.regionMode = true; } else if (cmd.equals("-zookeeper")) { this.zookeeperMode = true; } else if(cmd.equals("-regionserver")) { @@ -660,6 +707,8 @@ public final class Canary implements Tool { this.treatFailureAsError = true; } else if (cmd.equals("-e")) { this.useRegExp = true; + } else if (cmd.equals("-tableAvailabilitySummary")) { + this.tableAvailabilitySummary = true; } else if (cmd.equals("-t")) { i++; @@ -710,21 +759,24 @@ public final class Canary implements Tool { i++; if (i == args.length) { - System.err.println("-readTableTimeouts needs a comma-separated list of read timeouts per table (without spaces)."); + System.err.println("-readTableTimeouts needs a comma-separated list of " + + "read timeouts per table (without spaces)."); printUsageAndExit(); } String [] tableTimeouts = args[i].split(","); for (String tT: tableTimeouts) { String [] nameTimeout = tT.split("="); if (nameTimeout.length < 2) { - System.err.println("Each -readTableTimeouts argument must be of the form =."); + System.err.println("Each -readTableTimeouts argument must be of the form " + + "=."); printUsageAndExit(); } long timeoutVal = 0L; try { timeoutVal = Long.parseLong(nameTimeout[1]); } catch (NumberFormatException e) { - System.err.println("-readTableTimeouts read timeout for each table must be a numeric value argument."); + System.err.println("-readTableTimeouts read timeout for each table must be a numeric " + + "value argument."); printUsageAndExit(); } this.configuredReadTableTimeouts.put(nameTimeout[0], timeoutVal); @@ -750,7 +802,8 @@ public final class Canary implements Tool { printUsageAndExit(); } } - if (!this.configuredReadTableTimeouts.isEmpty() && (this.regionServerMode || this.zookeeperMode)) { + if (!this.configuredReadTableTimeouts.isEmpty() && + (this.regionServerMode || this.zookeeperMode)) { System.err.println("-readTableTimeouts can only be configured in region mode."); printUsageAndExit(); } @@ -815,7 +868,9 @@ public final class Canary implements Tool { return monitor.errorCode; } } finally { - if (monitor != null) monitor.close(); + if (monitor != null) { + monitor.close(); + } } Thread.sleep(interval); @@ -840,31 +895,42 @@ public final class Canary implements Tool { System.err.println( "Usage: hbase canary [opts] [table1 [table2]...] | [regionserver1 [regionserver2]..]"); System.err.println(" where [opts] are:"); - System.err.println(" -help Show this help and exit."); - System.err.println(" -regionserver replace the table argument to regionserver,"); - System.err.println(" which means to enable regionserver mode"); - System.err.println(" -allRegions Tries all regions on a regionserver,"); - System.err.println(" only works in regionserver mode."); - System.err.println(" -zookeeper Tries to grab zookeeper.znode.parent "); - System.err.println(" on each zookeeper instance"); - System.err.println(" -daemon Continuous check at defined intervals."); - System.err.println(" -interval Interval between checks (sec)"); - System.err.println(" -e Use table/regionserver as regular expression"); - System.err.println(" which means the table/regionserver is regular expression pattern"); - System.err.println(" -f stop whole program if first error occurs," + + System.err.println(" -help Show this help and exit."); + System.err.println(" -regionMode Run Canary in region mode."); + System.err.println(" -regionserver replace the table argument to regionserver,"); + System.err.println(" which means to enable regionserver mode"); + System.err.println(" -allRegions Tries all regions on a regionserver,"); + System.err.println(" only works in regionserver mode."); + System.err.println(" -zookeeper Tries to grab zookeeper.znode.parent "); + System.err.println(" on each zookeeper instance"); + System.err.println(" -daemon Continuous check at defined intervals."); + System.err.println(" -interval Interval between checks (sec)"); + System.err.println(" -e " + + " Use table/regionserver as regular expression"); + System.err.println(" which means the table/regionserver is regular " + + "expression pattern"); + System.err.println(" -f Stop whole program if first error occurs," + " default is true"); - System.err.println(" -t timeout for a check, default is 600000 (millisecs)"); - System.err.println(" -writeTableTimeout write timeout for the writeTable, default is 600000 (millisecs)"); - System.err.println(" -readTableTimeouts =,=, ... " - + "comma-separated list of read timeouts per table (no spaces), default is 600000 (millisecs)"); - System.err.println(" -writeSniffing enable the write sniffing in canary"); - System.err.println(" -treatFailureAsError treats read / write failure as error"); - System.err.println(" -writeTable The table used for write sniffing." + System.err.println(" -t Timeout for a check, default is 600000 " + + "(millisecs)"); + System.err.println(" -writeTableTimeout " + + " Write timeout for the writeTable, default is " + + "600000 (millisecs)"); + System.err.println(" -readTableTimeouts =,=" + + ", ... " + + "comma-separated list of read timeouts per table (no spaces), default is 600000" + + " (millisecs)"); + System.err.println(" -writeSniffing Enable the write sniffing in canary"); + System.err.println(" -treatFailureAsError Treats read / write failure as error"); + System.err.println(" -writeTable The table used for write sniffing." + " Default is hbase:canary"); - System.err.println(" -Dhbase.canary.read.raw.enabled= Use this flag to enable or disable raw scan during read canary test" - + " Default is false and raw is not enabled during scan"); + System.err.println(" -tableAvailabilitySummary Print table availability rate summary. " + + "Only for region mode."); + System.err.println(" -Dhbase.canary.read.raw.enabled= Use this flag to enable or " + + "disable raw scan during read canary test" + + " Default is false and raw is not enabled during scan"); System.err - .println(" -D= assigning or override the configuration params"); + .println(" -D= Assigning or override the configuration params"); System.exit(USAGE_EXIT_CODE); } @@ -884,22 +950,39 @@ public final class Canary implements Tool { monitorTargets = new String[length]; System.arraycopy(args, index, monitorTargets, 0, length); } - - if (this.sink instanceof RegionServerStdOutSink || this.regionServerMode) { + if (this.sink instanceof RegionStdOutSink || this.regionMode) { + if (! (this.sink instanceof RegionStdOutSink)) { + this.sink = new RegionStdOutSink(); + } + monitor = + new RegionMonitor(connection, monitorTargets, this.useRegExp, + (StdOutSink) this.sink, this.executor, this.writeSniffing, + this.writeTableName, this.treatFailureAsError, this.configuredReadTableTimeouts, + this.configuredWriteTableTimeout, this.tableAvailabilitySummary); + } else if (this.sink instanceof RegionServerStdOutSink || this.regionServerMode) { + if (! (this.sink instanceof RegionServerStdOutSink)) { + this.sink = new RegionServerStdOutSink(); + } monitor = new RegionServerMonitor(connection, monitorTargets, this.useRegExp, (StdOutSink) this.sink, this.executor, this.regionServerAllRegions, this.treatFailureAsError); } else if (this.sink instanceof ZookeeperStdOutSink || this.zookeeperMode) { + if (! (this.sink instanceof ZookeeperStdOutSink)) { + this.sink = new ZookeeperStdOutSink(); + } monitor = new ZookeeperMonitor(connection, monitorTargets, this.useRegExp, (StdOutSink) this.sink, this.executor, this.treatFailureAsError); } else { + if (! (this.sink instanceof RegionStdOutSink)) { + this.sink = new RegionStdOutSink(); + } monitor = new RegionMonitor(connection, monitorTargets, this.useRegExp, (StdOutSink) this.sink, this.executor, this.writeSniffing, this.writeTableName, this.treatFailureAsError, this.configuredReadTableTimeouts, - this.configuredWriteTableTimeout); + this.configuredWriteTableTimeout, this.tableAvailabilitySummary); } return monitor; } @@ -911,6 +994,7 @@ public final class Canary implements Tool { protected Admin admin; protected String[] targets; protected boolean useRegExp; + protected boolean tableAvailabilitySummary; protected boolean treatFailureAsError; protected boolean initialized = false; @@ -941,16 +1025,21 @@ public final class Canary implements Tool { @Override public void close() throws IOException { - if (this.admin != null) this.admin.close(); + if (this.admin != null) { + this.admin.close(); + } } protected Monitor(Connection connection, String[] monitorTargets, boolean useRegExp, Sink sink, - ExecutorService executor, boolean treatFailureAsError) { - if (null == connection) throw new IllegalArgumentException("connection shall not be null"); + ExecutorService executor, boolean treatFailureAsError, boolean tableAvailabilitySummary) { + if (null == connection) { + throw new IllegalArgumentException("connection shall not be null"); + } this.connection = connection; this.targets = monitorTargets; this.useRegExp = useRegExp; + this.tableAvailabilitySummary = tableAvailabilitySummary; this.treatFailureAsError = treatFailureAsError; this.sink = sink; this.executor = executor; @@ -995,8 +1084,11 @@ public final class Canary implements Tool { public RegionMonitor(Connection connection, String[] monitorTargets, boolean useRegExp, StdOutSink sink, ExecutorService executor, boolean writeSniffing, TableName writeTableName, - boolean treatFailureAsError, HashMap configuredReadTableTimeouts, long configuredWriteTableTimeout) { - super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError); + boolean treatFailureAsError, HashMap configuredReadTableTimeouts, + long configuredWriteTableTimeout, + boolean tableAvailabilitySummary) { + super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError, + tableAvailabilitySummary); Configuration conf = connection.getConfiguration(); this.writeSniffing = writeSniffing; this.writeTableName = writeTableName; @@ -1029,19 +1121,34 @@ public final class Canary implements Tool { RegionStdOutSink regionSink = this.getSink(); if (this.targets != null && this.targets.length > 0) { String[] tables = generateMonitorTables(this.targets); - // Check to see that each table name passed in the -readTableTimeouts argument is also passed as a monitor target. - if (! new HashSet<>(Arrays.asList(tables)).containsAll(this.configuredReadTableTimeouts.keySet())) { - LOG.error("-readTableTimeouts can only specify read timeouts for monitor targets passed via command line."); + // Check to see that each table name passed in the -readTableTimeouts argument is + // also passed as a monitor target. + if (! new HashSet<>(Arrays.asList(tables)) + .containsAll(this.configuredReadTableTimeouts.keySet())) { + LOG.error("-readTableTimeouts can only specify read timeouts for monitor targets " + + "passed via command line."); this.errorCode = USAGE_EXIT_CODE; return; } this.initialized = true; for (String table : tables) { LongAdder readLatency = regionSink.initializeAndGetReadLatencyForTable(table); + regionSink.initializeAndGetReadSuccessCountForTable(table); + regionSink.initializeAndGetReadFailureCountForTable(table); taskFutures.addAll(Canary.sniff(admin, regionSink, table, executor, TaskType.READ, this.rawScanEnabled, readLatency)); } - } else { + } else { // include all tables + Set tmpTables = new TreeSet<>(); + for(TableDescriptor t : admin.listTableDescriptors()) { + tmpTables.add(t.getTableName().getNameAsString()); + } + String[] tables = tmpTables.toArray(new String[tmpTables.size()]); + for (String table : tables) { + LongAdder readLatency = regionSink.initializeAndGetReadLatencyForTable(table); + regionSink.initializeAndGetReadSuccessCountForTable(table); + regionSink.initializeAndGetReadFailureCountForTable(table); + } taskFutures.addAll(sniff(TaskType.READ, regionSink)); } @@ -1057,7 +1164,8 @@ public final class Canary implements Tool { // sniff canary table with write operation regionSink.initializeWriteLatency(); LongAdder writeTableLatency = regionSink.getWriteLatency(); - taskFutures.addAll(Canary.sniff(admin, regionSink, admin.getTableDescriptor(writeTableName), + taskFutures.addAll( + Canary.sniff(admin, regionSink, admin.getTableDescriptor(writeTableName), executor, TaskType.WRITE, this.rawScanEnabled, writeTableLatency)); } @@ -1077,7 +1185,8 @@ public final class Canary implements Tool { LOG.info("Read operation for " + tableName + " took " + actual + " ms. The configured read timeout was " + configured + " ms."); if (actual > configured) { - LOG.error("Read operation for " + tableName + " exceeded the configured read timeout."); + LOG.error("Read operation for " + tableName + " exceeded the configured read " + + "timeout."); } } else { LOG.error("Read operation for " + tableName + " failed!"); @@ -1086,11 +1195,14 @@ public final class Canary implements Tool { if (this.writeSniffing) { String writeTableStringName = this.writeTableName.getNameAsString(); long actualWriteLatency = regionSink.getWriteLatency().longValue(); - LOG.info("Write operation for " + writeTableStringName + " took " + actualWriteLatency + " ms. The configured write timeout was " + + LOG.info("Write operation for " + writeTableStringName + " took " + + actualWriteLatency + " ms. The configured write timeout was " + this.configuredWriteTableTimeout + " ms."); - // Check that the writeTable write operation latency does not exceed the configured timeout. + // Check that the writeTable write operation latency does not exceed the configured + // timeout. if (actualWriteLatency > this.configuredWriteTableTimeout) { - LOG.error("Write operation for " + writeTableStringName + " exceeded the configured write timeout."); + LOG.error("Write operation for " + writeTableStringName + " exceeded the " + + "configured write timeout."); } } } catch (Exception e) { @@ -1098,7 +1210,20 @@ public final class Canary implements Tool { this.errorCode = ERROR_EXIT_CODE; } finally { this.done = true; - } + } + } + if (this.tableAvailabilitySummary) { + LOG.info("========================================"); + LOG.info("=============== Summary: ==============="); + + for (Map.Entry entry : + this.getSink().perTableReadSuccessCount.entrySet()) { + LOG.info("Read success rate for table : " + entry.getKey() + " is: " + + (double)(entry.getValue().longValue()) / + (entry.getValue().longValue() + this.getSink().perTableReadFailureCount + .get(entry.getKey()).longValue())); + } + LOG.info("===============END======================"); } this.done = true; } @@ -1149,7 +1274,8 @@ public final class Canary implements Tool { /* * canary entry point to monitor all the tables. */ - private List> sniff(TaskType taskType, RegionStdOutSink regionSink) throws Exception { + private List> sniff(TaskType taskType, RegionStdOutSink regionSink) + throws Exception { if (LOG.isDebugEnabled()) { LOG.debug(String.format("reading list of tables")); } @@ -1157,8 +1283,10 @@ public final class Canary implements Tool { for (HTableDescriptor table : admin.listTables()) { if (admin.isTableEnabled(table.getTableName()) && (!table.getTableName().equals(writeTableName))) { - LongAdder readLatency = regionSink.initializeAndGetReadLatencyForTable(table.getNameAsString()); - taskFutures.addAll(Canary.sniff(admin, sink, table, executor, taskType, this.rawScanEnabled, readLatency)); + LongAdder readLatency = regionSink.initializeAndGetReadLatencyForTable( + table.getNameAsString()); + taskFutures.addAll(Canary.sniff(admin, sink, table, executor, taskType, + this.rawScanEnabled, readLatency)); } } return taskFutures; @@ -1227,7 +1355,8 @@ public final class Canary implements Tool { * @throws Exception */ private static List> sniff(final Admin admin, final Sink sink, String tableName, - ExecutorService executor, TaskType taskType, boolean rawScanEnabled, LongAdder readLatency) throws Exception { + ExecutorService executor, TaskType taskType, boolean rawScanEnabled, LongAdder readLatency) + throws Exception { if (LOG.isDebugEnabled()) { LOG.debug(String.format("checking table is enabled and getting table descriptor for table %s", tableName)); @@ -1271,7 +1400,8 @@ public final class Canary implements Tool { for (HRegionLocation location : regionLocator.getAllRegionLocations()) { ServerName rs = location.getServerName(); RegionInfo region = location.getRegionInfo(); - tasks.add(new RegionTask(admin.getConnection(), region, rs, (RegionStdOutSink) sink, taskType, rawScanEnabled, + tasks.add(new RegionTask(admin.getConnection(), region, rs, (RegionStdOutSink) sink, + taskType, rawScanEnabled, rwLatency)); } } finally { @@ -1290,7 +1420,7 @@ public final class Canary implements Tool { protected ZookeeperMonitor(Connection connection, String[] monitorTargets, boolean useRegExp, StdOutSink sink, ExecutorService executor, boolean treatFailureAsError) { - super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError); + super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError, false); Configuration configuration = connection.getConfiguration(); znode = configuration.get(ZOOKEEPER_ZNODE_PARENT, @@ -1352,7 +1482,7 @@ public final class Canary implements Tool { public RegionServerMonitor(Connection connection, String[] monitorTargets, boolean useRegExp, StdOutSink sink, ExecutorService executor, boolean allRegions, boolean treatFailureAsError) { - super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError); + super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError, false); this.allRegions = allRegions; } @@ -1395,7 +1525,9 @@ public final class Canary implements Tool { return false; } - if (this.targets == null || this.targets.length == 0) return true; + if (this.targets == null || this.targets.length == 0) { + return true; + } for (String target : this.targets) { for (TableName tableName : tableNames) { @@ -1413,7 +1545,8 @@ public final class Canary implements Tool { return foundTableNames.isEmpty(); } - private void monitorRegionServers(Map> rsAndRMap, RegionServerStdOutSink regionServerSink) { + private void monitorRegionServers(Map> rsAndRMap, + RegionServerStdOutSink regionServerSink) { List tasks = new ArrayList<>(); Map successMap = new HashMap<>(); Random rand = new Random(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java index e713a5af81..3b6de70ad8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java @@ -117,13 +117,25 @@ public class TestCanaryTool { p.addColumn(FAMILY, COLUMN, iBytes); table.put(p); } + double epsilon = 0.0001; ExecutorService executor = new ScheduledThreadPoolExecutor(1); Canary.RegionStdOutSink sink = spy(new Canary.RegionStdOutSink()); Canary canary = new Canary(executor, sink); String[] args = { "-writeSniffing", "-t", "10000", name.getMethodName() }; assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args)); - assertEquals("verify no read error count", 0, canary.getReadFailures().size()); - assertEquals("verify no write error count", 0, canary.getWriteFailures().size()); + assertEquals("verify no read error count", 0, + canary.getReadFailures().size()); + assertEquals("verify no write error count", 0, + canary.getWriteFailures().size()); + assertEquals("perTableReadSuccessCount ", 1, + sink.getPerTableReadSuccessCount().size()); + assertEquals("perTableReadFailureCount count ", 1, + sink.getPerTableReadFailureCount().size()); + assertEquals("testBasicCanaryWorks", table.getName().toString()); + assertEquals(1.0, sink.getPerTableReadSuccessCount().get(table.getName().toString()) + .doubleValue(), epsilon); + assertEquals(0.0, sink.getPerTableReadFailureCount().get(table.getName().toString()) + .doubleValue(), epsilon); verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(HRegionInfo.class), isA(ColumnFamilyDescriptor.class), anyLong()); } -- 2.14.3 (Apple Git-98)