From 7aa09930627146267d52e734ecd1d4787fb21ddb Mon Sep 17 00:00:00 2001 From: Yechao Chen Date: Wed, 28 Feb 2018 13:59:11 +0800 Subject: [PATCH] HBASE-20056 Performance optimization on MultiTableInputFormatBase#getSplits() --- .../hbase/mapreduce/MultiTableInputFormatBase.java | 105 +++++++++++---------- 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java index 6f0075a2e2..0365ab40b8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java @@ -181,58 +181,61 @@ public abstract class MultiTableInputFormatBase extends List splits = new ArrayList(); Iterator iter = tableMaps.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry> entry = (Map.Entry>) iter.next(); - TableName tableName = entry.getKey(); - List scanList = entry.getValue(); - - try (Connection conn = ConnectionFactory.createConnection(context.getConfiguration()); - Table table = conn.getTable(tableName); - RegionLocator regionLocator = conn.getRegionLocator(tableName)) { - RegionSizeCalculator sizeCalculator = new RegionSizeCalculator( - regionLocator, conn.getAdmin()); - Pair keys = regionLocator.getStartEndKeys(); - for (Scan scan : scanList) { - if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { - throw new IOException("Expecting at least one region for table : " - + tableName.getNameAsString()); - } - int count = 0; - - byte[] startRow = scan.getStartRow(); - byte[] stopRow = scan.getStopRow(); - - for (int i = 0; i < keys.getFirst().length; i++) { - if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) { - continue; + // Make a single Connection to the Cluster and use it across all tables. + try (Connection conn = ConnectionFactory.createConnection(context.getConfiguration())) { + while (iter.hasNext()) { + Map.Entry> entry = (Map.Entry>) iter.next(); + TableName tableName = entry.getKey(); + List scanList = entry.getValue(); + + try (Table table = conn.getTable(tableName); + RegionLocator regionLocator = conn.getRegionLocator(tableName)) { + RegionSizeCalculator sizeCalculator = new RegionSizeCalculator( + regionLocator, conn.getAdmin()); + Pair keys = regionLocator.getStartEndKeys(); + for (Scan scan : scanList) { + if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { + throw new IOException("Expecting at least one region for table : " + + tableName.getNameAsString()); } - - if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || - Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) && - (stopRow.length == 0 || Bytes.compareTo(stopRow, - keys.getFirst()[i]) > 0)) { - byte[] splitStart = startRow.length == 0 || - Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? - keys.getFirst()[i] : startRow; - byte[] splitStop = (stopRow.length == 0 || - Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) && - keys.getSecond()[i].length > 0 ? - keys.getSecond()[i] : stopRow; - - HRegionLocation hregionLocation = regionLocator.getRegionLocation( - keys.getFirst()[i], false); - String regionHostname = hregionLocation.getHostname(); - HRegionInfo regionInfo = hregionLocation.getRegionInfo(); - long regionSize = sizeCalculator.getRegionSize( - regionInfo.getRegionName()); - - TableSplit split = new TableSplit(table.getName(), - scan, splitStart, splitStop, regionHostname, regionSize); - - splits.add(split); - - if (LOG.isDebugEnabled()) - LOG.debug("getSplits: split -> " + (count++) + " -> " + split); + int count = 0; + + byte[] startRow = scan.getStartRow(); + byte[] stopRow = scan.getStopRow(); + + for (int i = 0; i < keys.getFirst().length; i++) { + if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) { + continue; + } + + if ((startRow.length == 0 || keys.getSecond()[i].length == 0 || + Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) && + (stopRow.length == 0 || Bytes.compareTo(stopRow, + keys.getFirst()[i]) > 0)) { + byte[] splitStart = startRow.length == 0 || + Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? + keys.getFirst()[i] : startRow; + byte[] splitStop = (stopRow.length == 0 || + Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0) && + keys.getSecond()[i].length > 0 ? + keys.getSecond()[i] : stopRow; + + HRegionLocation hregionLocation = regionLocator.getRegionLocation( + keys.getFirst()[i], false); + String regionHostname = hregionLocation.getHostname(); + HRegionInfo regionInfo = hregionLocation.getRegionInfo(); + long regionSize = sizeCalculator.getRegionSize( + regionInfo.getRegionName()); + + TableSplit split = new TableSplit(table.getName(), + scan, splitStart, splitStop, regionHostname, regionSize); + + splits.add(split); + + if (LOG.isDebugEnabled()) { + LOG.debug("getSplits: split -> " + (count++) + " -> " + split); + } + } } } } -- 2.15.0.windows.1