Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteColumnsByPrefix.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteColumnsByPrefix.java (revision 0) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteColumnsByPrefix.java (revision 0) @@ -0,0 +1,244 @@ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestDeleteColumnsByPrefix { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static String FAMILY_STR = "family"; + private final static byte[] FAMILY = Bytes.toBytes(FAMILY_STR); + private final static byte[] ROW = Bytes.toBytes("row"); + + @Test + public void testBasicCases() throws IOException { + HTableDescriptor htd = new HTableDescriptor("TestDeleteColumnsByPrefix"); + htd.addFamily(new HColumnDescriptor(FAMILY_STR)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + + long T = EnvironmentEdgeManager.currentTimeMillis(); + Put p = new Put(ROW, T); + p.add(FAMILY, Bytes.toBytes("12"), ROW); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + p.add(FAMILY, Bytes.toBytes("1234"), ROW); + p.add(FAMILY, Bytes.toBytes("12345"), ROW); + p.add(FAMILY, Bytes.toBytes("125"), ROW); + p.add(FAMILY, Bytes.toBytes("126"), ROW); + region.put(p); + + Delete d = new Delete(ROW); + d.deleteColumnsByPrefix(FAMILY, Bytes.toBytes("123"), T+1); + region.delete(d, null, false); + Scan s = new Scan(); + RegionScanner scanner = region.getScanner(s); + List kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(3, kvs.size()); + // 123 is not a prefix of 12, 125, or 126, so these kvs is not affected + assertArrayEquals(Bytes.toBytes("12"), kvs.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(1).getQualifier()); + assertArrayEquals(Bytes.toBytes("126"), kvs.get(2).getQualifier()); + scanner.close(); + + p = new Put(ROW, T+2); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + region.put(p); + + s = new Scan(); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + // new kv, not affected + assertArrayEquals(Bytes.toBytes("12"), kvs.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("123"), kvs.get(1).getQualifier()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(2).getQualifier()); + assertArrayEquals(Bytes.toBytes("126"), kvs.get(3).getQualifier()); + scanner.close(); + + // flush + region.flushcache(); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + scanner.close(); + + // compact + region.compactStores(false); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + scanner.close(); + + Scan s1 = new Scan(); + s1.setRaw(true); + scanner = region.getScanner(s1); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(5, kvs.size()); + assertArrayEquals(Bytes.toBytes("12"), kvs.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("123"), kvs.get(1).getQualifier()); + assertTrue(kvs.get(2).isDelete()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(3).getQualifier()); + assertArrayEquals(Bytes.toBytes("126"), kvs.get(4).getQualifier()); + scanner.close(); + + region.compactStores(true); + scanner = region.getScanner(s1); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + scanner.close(); + + region.close(); + region.getLog().closeAndDelete(); + } + + @Test + public void testMultiplePrefixMarkers() throws IOException { + HTableDescriptor htd = new HTableDescriptor("TestDeleteColumnsByPrefix"); + htd.addFamily(new HColumnDescriptor(FAMILY_STR)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + + long T = EnvironmentEdgeManager.currentTimeMillis(); + Put p = new Put(ROW, T); + p.add(FAMILY, Bytes.toBytes("12"), ROW); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + p.add(FAMILY, Bytes.toBytes("1234"), ROW); + region.put(p); + + Delete d = new Delete(ROW); + d.deleteColumnsByPrefix(FAMILY, Bytes.toBytes("123"), T+1); + region.delete(d, null, false); + + // place an earlier marker + d = new Delete(ROW); + d.deleteColumnsByPrefix(FAMILY, Bytes.toBytes("123"), T-1); + region.delete(d, null, false); + + d = new Delete(ROW); + d.deleteColumnsByPrefix(FAMILY, Bytes.toBytes("1234"), T-1); + region.delete(d, null, false); + + Scan s = new Scan(); + RegionScanner scanner = region.getScanner(s); + List kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(1, kvs.size()); + + p = new Put(ROW, T+2); + p.add(FAMILY, Bytes.toBytes("1234"), ROW); + region.put(p); + d = new Delete(ROW); + d.deleteColumnsByPrefix(FAMILY, Bytes.toBytes("1234"), T+3); + region.delete(d, null, false); + s = new Scan(); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(1, kvs.size()); + } + + @Test + public void testMixedMarkers() throws IOException { + HTableDescriptor htd = new HTableDescriptor("TestDeleteColumnsByPrefix"); + htd.addFamily(new HColumnDescriptor(FAMILY_STR)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + + long T = EnvironmentEdgeManager.currentTimeMillis(); + Put p = new Put(ROW, T); + p.add(FAMILY, Bytes.toBytes("12"), ROW); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + p.add(FAMILY, Bytes.toBytes("1234"), ROW); + p.add(FAMILY, Bytes.toBytes("12345"), ROW); + p.add(FAMILY, Bytes.toBytes("125"), ROW); + region.put(p); + + Delete d = new Delete(ROW); + d.deleteColumnsByPrefix(FAMILY, Bytes.toBytes("123"), T+1); + region.delete(d, null, false); + + // a new version of column "123" + p = new Put(ROW, T+2); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + region.put(p); + + // now delete it with a column marker + d = new Delete(ROW); + d.deleteColumns(FAMILY, Bytes.toBytes("123"), T+3); + d.deleteColumns(FAMILY, Bytes.toBytes("12"), T+3); + region.delete(d, null, false); + + Scan s = new Scan(); + s.setRaw(true); + RegionScanner scanner = region.getScanner(s); + List kvs = new ArrayList(); + scanner.next(kvs); + scanner.next(kvs); + assertEquals(8, kvs.size()); + assertTrue(kvs.get(0).isDelete()); // delete for 12 + assertArrayEquals(Bytes.toBytes("12"), kvs.get(1).getQualifier()); + assertTrue(kvs.get(2).isDelete()); // delete for 123 + assertArrayEquals(Bytes.toBytes("123"), kvs.get(3).getQualifier()); + assertTrue(kvs.get(4).isDelete()); // delete prefix for 123 + assertArrayEquals(Bytes.toBytes("1234"), kvs.get(5).getQualifier()); + assertArrayEquals(Bytes.toBytes("12345"), kvs.get(6).getQualifier()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(7).getQualifier()); + scanner.close(); + + s = new Scan(); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + // the following is... strange + // we happen get to this order: + // ... + // column delete for 123 at T+3 + // put for 123 at T+2 + // column prefix delete for prefix 123 at T+1 + // ... + // Now the ScanDeleteTracker sees the put for 123 and determines that + // it is deleted (due to the column marker at T+3), it will skip all + // 123 columns *including* the prefix delete marker at T+1. + // Thus the puts for 1234 and 12345 are now visible again. + // Should not mix deleteColumnsByPrefix and deleteColumns with the same + // qualifier. + assertEquals(3, kvs.size()); + // this should only see 125 + assertArrayEquals(Bytes.toBytes("1234"), kvs.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("12345"), kvs.get(1).getQualifier()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(2).getQualifier()); + scanner.close(); + + region.close(); + region.getLog().closeAndDelete(); + } +} Index: src/main/java/org/apache/hadoop/hbase/client/Delete.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/Delete.java (revision 1235445) +++ src/main/java/org/apache/hadoop/hbase/client/Delete.java (working copy) @@ -202,6 +202,43 @@ } /** + * Delete all versions of all columns that match the specified qualifier + * prefix. + * Note: Do not mix deleteColumnsByPrefix and + * {@link #deleteColumns(byte[], byte[])} for the same qualifier + * @param family family name + * @param qualifier column qualifier prefix to match + * @return this for invocation chaining + */ + public Delete deleteColumnsByPrefix(byte[] family, byte[] qualifier) { + this.deleteColumnsByPrefix(family, qualifier, + HConstants.LATEST_TIMESTAMP); + return this; + } + + /** + * Delete all versions of all columns that match the specified qualifier prefix + * with a timestamp less than r equal to the specified timestamp. + * Note: Do not mix deleteColumnsByPrefix and + * {@link #deleteColumns(byte[], byte[])} for the same qualifier + * @param family family name + * @param qualifier column qualifier prefix to match + * @param timestamp maximum version timestamp + * @return this for invocation chaining + */ + public Delete deleteColumnsByPrefix(byte[] family, byte[] qualifier, + long timestamp) { + List list = familyMap.get(family); + if (list == null) { + list = new ArrayList(); + } + list.add(new KeyValue(this.row, family, qualifier, timestamp, + KeyValue.Type.DeleteColumnPrefix)); + familyMap.put(family, list); + return this; + } + + /** * Delete all versions of the specified column. * @param family family name * @param qualifier column qualifier Index: src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java (revision 1235445) +++ src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java (working copy) @@ -113,23 +113,36 @@ int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, buffer, qualifierOffset, qualifierLength); - if (ret == 0) { - if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { - return DeleteResult.COLUMN_DELETED; + if (ret <= 0) { + // check whether the delete marker is a prefix + if (deleteType == KeyValue.Type.DeleteColumnPrefix.getCode()) { + if (Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, buffer, + qualifierOffset, deleteLength) == 0) { + if (timestamp <= deleteTimestamp) { + return DeleteResult.COLUMN_DELETED; + } + } else { + // past the prefix marker + deleteBuffer = null; + } + } else if (ret == 0) { + if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { + return DeleteResult.COLUMN_DELETED; + } + // Delete (aka DeleteVersion) + // If the timestamp is the same, keep this one + if (timestamp == deleteTimestamp) { + return DeleteResult.VERSION_DELETED; + } + // use assert or not? + assert timestamp < deleteTimestamp; + + // different timestamp, let's clear the buffer. + deleteBuffer = null; + } else if(ret < 0){ + // Next column case. + deleteBuffer = null; } - // Delete (aka DeleteVersion) - // If the timestamp is the same, keep this one - if (timestamp == deleteTimestamp) { - return DeleteResult.VERSION_DELETED; - } - // use assert or not? - assert timestamp < deleteTimestamp; - - // different timestamp, let's clear the buffer. - deleteBuffer = null; - } else if(ret < 0){ - // Next column case. - deleteBuffer = null; } else { throw new IllegalStateException("isDelete failed: deleteBuffer=" + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) Index: src/main/java/org/apache/hadoop/hbase/KeyValue.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/KeyValue.java (revision 1235445) +++ src/main/java/org/apache/hadoop/hbase/KeyValue.java (working copy) @@ -161,6 +161,7 @@ Delete((byte)8), DeleteColumn((byte)12), + DeleteColumnPrefix((byte)13), DeleteFamily((byte)14), // Maximum is used when searching; you look from maximum on down.