From 4f1e33c9296b1bb85c68df330fc4f24663ad6512 Mon Sep 17 00:00:00 2001 From: Matt Warhaftig Date: Sun, 3 Apr 2016 18:41:42 -0400 Subject: [PATCH] HBASE-15287 Allow for binary row values in HBase MapReduce jobs. --- .../main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java | 2 +- .../main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java | 2 +- .../src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java | 4 ++-- .../src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java | 6 +++--- .../java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java | 2 +- .../src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java | 4 ++-- .../apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java | 2 +- .../java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java | 4 ++-- .../test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java | 5 +++-- 9 files changed, 16 insertions(+), 15 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java index 6cd0602..ee6da75 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java @@ -154,6 +154,6 @@ implements TableMap { } sb.append(Bytes.toString(vals[i])); } - return new ImmutableBytesWritable(Bytes.toBytes(sb.toString())); + return new ImmutableBytesWritable(Bytes.toBytesBinary(sb.toString())); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java index 00f197c..aaa32bd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java @@ -241,7 +241,7 @@ public class CellCounter extends Configured implements Tool { String regexPattern = filterCriteria.substring(1, filterCriteria.length()); rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regexPattern)); } else { - rowFilter = new PrefixFilter(Bytes.toBytes(filterCriteria)); + rowFilter = new PrefixFilter(Bytes.toBytesBinary(filterCriteria)); } return rowFilter; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java index dd8b891..c1e8a82 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java @@ -111,11 +111,11 @@ public class CopyTable extends Configured implements Tool { } if (startRow != null) { - scan.setStartRow(Bytes.toBytes(startRow)); + scan.setStartRow(Bytes.toBytesBinary(startRow)); } if (stopRow != null) { - scan.setStopRow(Bytes.toBytes(stopRow)); + scan.setStopRow(Bytes.toBytesBinary(stopRow)); } if(families != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java index 66c0057..56d229a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java @@ -100,10 +100,10 @@ public class Export extends Configured implements Tool { s.setCacheBlocks(false); // set Start and Stop row if (conf.get(TableInputFormat.SCAN_ROW_START) != null) { - s.setStartRow(Bytes.toBytes(conf.get(TableInputFormat.SCAN_ROW_START))); + s.setStartRow(Bytes.toBytesBinary(conf.get(TableInputFormat.SCAN_ROW_START))); } if (conf.get(TableInputFormat.SCAN_ROW_STOP) != null) { - s.setStopRow(Bytes.toBytes(conf.get(TableInputFormat.SCAN_ROW_STOP))); + s.setStopRow(Bytes.toBytesBinary(conf.get(TableInputFormat.SCAN_ROW_STOP))); } // Set Scan Column Family boolean raw = Boolean.parseBoolean(conf.get(RAW_SCAN)); @@ -142,7 +142,7 @@ public class Export extends Configured implements Tool { String regexPattern = filterCriteria.substring(1, filterCriteria.length()); exportFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator(regexPattern)); } else { - exportFilter = new PrefixFilter(Bytes.toBytes(filterCriteria)); + exportFilter = new PrefixFilter(Bytes.toBytesBinary(filterCriteria)); } return exportFilter; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java index e9c8927..8a9fa49 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/GroupingTableMapper.java @@ -145,7 +145,7 @@ extends TableMapper implements Configurable { } sb.append(Bytes.toString(vals[i])); } - return new ImmutableBytesWritable(Bytes.toBytes(sb.toString())); + return new ImmutableBytesWritable(Bytes.toBytesBinary(sb.toString())); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java index 8522a61..720d4b1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java @@ -147,10 +147,10 @@ public class RowCounter extends Configured implements Tool { Scan scan = new Scan(); scan.setCacheBlocks(false); if (startKey != null && !startKey.equals("")) { - scan.setStartRow(Bytes.toBytes(startKey)); + scan.setStartRow(Bytes.toBytesBinary(startKey)); } if (endKey != null && !endKey.equals("")) { - scan.setStopRow(Bytes.toBytes(endKey)); + scan.setStopRow(Bytes.toBytesBinary(endKey)); } if (sb.length() > 0) { for (String columnName : sb.toString().trim().split(" ")) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java index fe967ff..2257054 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/SimpleTotalOrderPartitioner.java @@ -94,7 +94,7 @@ implements Configurable { } LOG.warn("Using deprecated configuration " + deprecatedKey + " - please use static accessor methods instead."); - return Bytes.toBytes(oldStyleVal); + return Bytes.toBytesBinary(oldStyleVal); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java index 814d82c..ebeb158 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java @@ -129,11 +129,11 @@ implements Configurable { scan = new Scan(); if (conf.get(SCAN_ROW_START) != null) { - scan.setStartRow(Bytes.toBytes(conf.get(SCAN_ROW_START))); + scan.setStartRow(Bytes.toBytesBinary(conf.get(SCAN_ROW_START))); } if (conf.get(SCAN_ROW_STOP) != null) { - scan.setStopRow(Bytes.toBytes(conf.get(SCAN_ROW_STOP))); + scan.setStopRow(Bytes.toBytesBinary(conf.get(SCAN_ROW_STOP))); } if (conf.get(SCAN_COLUMNS) != null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java index 6657d0f..4a719dc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java @@ -149,7 +149,7 @@ public class TestRowCounter { @Test public void testRowCounterColumnAndRowRange() throws Exception { String[] args = new String[] { - TABLE_NAME, "--range=rov,rox", COL_FAM + ":" + COL1 + TABLE_NAME, "--range=\\x00rov,\\x00rox", COL_FAM + ":" + COL1 }; runRowCount(args, 8); } @@ -245,7 +245,8 @@ public class TestRowCounter { // write few rows with two columns int i = 0; for (; i < TOTAL_ROWS - ROWS_WITH_ONE_COL; i++) { - byte[] row = Bytes.toBytes("row" + i); + // Use binary rows values to test for HBASE-15287. + byte[] row = Bytes.toBytesBinary("\\x00row" + i); Put put = new Put(row); put.addColumn(family, col1, value); put.addColumn(family, col2, value); -- 2.5.4 (Apple Git-61)