From dc6c4bacdc563269bf87eb34b566b6c4b92db73e Mon Sep 17 00:00:00 2001 From: jingyuntian Date: Thu, 21 Jun 2018 20:02:00 +0800 Subject: [PATCH] HBASE-20769 getSplits() has a out of bounds problem in TableSnapshotInputFormatImpl --- .../hbase/mapreduce/TableSnapshotInputFormat.java | 4 +++ .../mapreduce/TableSnapshotInputFormatImpl.java | 6 ++-- .../mapreduce/TestTableSnapshotInputFormat.java | 33 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java index d3e0d56..1b6f4f8 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java @@ -137,6 +137,10 @@ public class TableSnapshotInputFormat extends InputFormat 0 ? scan.getStartRow() : sp[i]); + boundedScan.setStopRow( + Bytes.compareTo(scan.getStopRow(), sp[i + 1]) < 0 ? scan.getStopRow() : sp[i + 1]); splits.add(new InputSplit(htd, hri, hosts, boundedScan, restoreDir)); } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java index ac9862d..7d69733 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java @@ -76,6 +76,8 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa private static final byte[] bbb = Bytes.toBytes("bbb"); private static final byte[] yyy = Bytes.toBytes("yyy"); + private static final byte[] bbc = Bytes.toBytes("bbc"); + private static final byte[] yya = Bytes.toBytes("yya"); @Rule public TestName name = new TestName(); @@ -247,6 +249,32 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa } @Test + public void testWithMockedMapReduceWithSplitsPerRegion() throws Exception { + setupCluster(); + String snapshotName = "testWithMockedMapReduceMultiRegion"; + final TableName tableName = TableName.valueOf(name.getMethodName()); + try { + createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 10); + + Configuration conf = UTIL.getConfiguration(); + conf.setBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_KEY, false); + Job job = new Job(conf); + Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName); + Scan scan = new Scan(bbc, yya); // limit the scan + + TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, scan, + TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, false, + tmpTableDir, new RegionSplitter.UniformSplit(), 5); + + verifyWithMockedMapReduce(job, 10, 40, bbc, yya); + } finally { + UTIL.getAdmin().deleteSnapshot(snapshotName); + UTIL.deleteTable(tableName); + tearDownCluster(); + } + } + + @Test public void testNoDuplicateResultsWhenSplitting() throws Exception { setupCluster(); TableName tableName = TableName.valueOf("testNoDuplicateResultsWhenSplitting"); @@ -317,12 +345,17 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa // validate input split InputSplit split = splits.get(i); Assert.assertTrue(split instanceof TableSnapshotRegionSplit); + TableSnapshotRegionSplit snapshotRegionSplit = (TableSnapshotRegionSplit) split; if (localityEnabled) { Assert.assertTrue(split.getLocations() != null && split.getLocations().length != 0); } else { Assert.assertTrue(split.getLocations() != null && split.getLocations().length == 0); } + Scan scan = TableMapReduceUtil.convertStringToScan(snapshotRegionSplit.getDelegate().getScan()); + Assert.assertTrue(Bytes.toStringBinary(startRow) + " <= "+ Bytes.toStringBinary(scan.getStartRow()) + "?", Bytes.compareTo(startRow, scan.getStartRow()) <= 0); + Assert.assertTrue(Bytes.toStringBinary(stopRow) + " >= "+ Bytes.toStringBinary(scan.getStopRow()) + "?", Bytes.compareTo(stopRow, scan.getStopRow()) >= 0); + // validate record reader TaskAttemptContext taskAttemptContext = mock(TaskAttemptContext.class); when(taskAttemptContext.getConfiguration()).thenReturn(job.getConfiguration()); -- 2.7.4