Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteColumnsByPrefix.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteColumnsByPrefix.java (revision 0) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteColumnsByPrefix.java (revision 0) @@ -0,0 +1,214 @@ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestDeleteColumnsByPrefix { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static String FAMILY_STR = "family"; + private final static byte[] FAMILY = Bytes.toBytes(FAMILY_STR); + private final static byte[] ROW = Bytes.toBytes("row"); + + @Test + public void testBasicCases() throws IOException { + HTableDescriptor htd = new HTableDescriptor("TestDeleteColumnsByPrefix"); + htd.addFamily(new HColumnDescriptor(FAMILY_STR)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + + long T = EnvironmentEdgeManager.currentTimeMillis(); + Put p = new Put(ROW, T); + p.add(FAMILY, Bytes.toBytes("12"), ROW); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + p.add(FAMILY, Bytes.toBytes("1234"), ROW); + p.add(FAMILY, Bytes.toBytes("12345"), ROW); + p.add(FAMILY, Bytes.toBytes("125"), ROW); + p.add(FAMILY, Bytes.toBytes("126"), ROW); + region.put(p); + + Delete d = new Delete(ROW); + d.deleteColumnsByPrefix(FAMILY, Bytes.toBytes("123"), T+1); + region.delete(d, null, false); + Scan s = new Scan(); + RegionScanner scanner = region.getScanner(s); + List kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(3, kvs.size()); + // 123 is not a prefix of 12, 125, or 126, so these kvs is not affected + assertArrayEquals(Bytes.toBytes("12"), kvs.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(1).getQualifier()); + assertArrayEquals(Bytes.toBytes("126"), kvs.get(2).getQualifier()); + scanner.close(); + + p = new Put(ROW, T+2); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + region.put(p); + + s = new Scan(); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + // new kv, not affected + assertArrayEquals(Bytes.toBytes("12"), kvs.get(0).getQualifier()); + assertArrayEquals(Bytes.toBytes("123"), kvs.get(1).getQualifier()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(2).getQualifier()); + assertArrayEquals(Bytes.toBytes("126"), kvs.get(3).getQualifier()); + scanner.close(); + + // flush + region.flushcache(); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + scanner.close(); + + // compact + region.compactStores(false); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + scanner.close(); + + Scan s1 = new Scan(); + s1.setRaw(true); + scanner = region.getScanner(s1); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(5, kvs.size()); + assertArrayEquals(Bytes.toBytes("12"), kvs.get(0).getQualifier()); + assertTrue(kvs.get(1).isDelete()); + assertArrayEquals(Bytes.toBytes("123"), kvs.get(2).getQualifier()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(3).getQualifier()); + assertArrayEquals(Bytes.toBytes("126"), kvs.get(4).getQualifier()); + scanner.close(); + + region.compactStores(true); + scanner = region.getScanner(s1); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(4, kvs.size()); + scanner.close(); + + region.close(); + region.getLog().closeAndDelete(); + } + + @Test + public void testStuff() throws IOException { + HTableDescriptor htd = new HTableDescriptor("TestDeleteColumnsByPrefix"); + htd.addFamily(new HColumnDescriptor(FAMILY_STR)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + + long T = EnvironmentEdgeManager.currentTimeMillis(); + Put p = new Put(ROW, T); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + region.put(p); + + Delete d = new Delete(ROW); + d.deleteColumns(FAMILY, Bytes.toBytes("123"), T+1); + region.delete(d, null, false); + + // add a new row + p = new Put(ROW, T+2); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + region.put(p); + // delete that exact version + d = new Delete(ROW); + d.deleteColumn(FAMILY, Bytes.toBytes("123"), T+2); + region.delete(d, null, false); + Scan s = new Scan(); + //s.setRaw(true); + RegionScanner scanner = region.getScanner(s); + List kvs = new ArrayList(); + scanner.next(kvs); + assertTrue(kvs.isEmpty()); + } + + @Test + public void testMixedMarkers() throws IOException { + HTableDescriptor htd = new HTableDescriptor("TestDeleteColumnsByPrefix"); + htd.addFamily(new HColumnDescriptor(FAMILY_STR)); + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + HRegion region = HRegion.createHRegion(info, TEST_UTIL. + getDataTestDir(), TEST_UTIL.getConfiguration(), htd); + + long T = EnvironmentEdgeManager.currentTimeMillis(); + Put p = new Put(ROW, T); + p.add(FAMILY, Bytes.toBytes("12"), ROW); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + p.add(FAMILY, Bytes.toBytes("1234"), ROW); + p.add(FAMILY, Bytes.toBytes("12345"), ROW); + p.add(FAMILY, Bytes.toBytes("125"), ROW); + region.put(p); + + Delete d = new Delete(ROW); + d.deleteColumnsByPrefix(FAMILY, Bytes.toBytes("123"), T+1); + region.delete(d, null, false); + + // a new version of column "123" + p = new Put(ROW, T+2); + p.add(FAMILY, Bytes.toBytes("123"), ROW); + region.put(p); + + // now delete it with a column marker + d = new Delete(ROW); + d.deleteColumns(FAMILY, Bytes.toBytes("123"), T+3); + d.deleteColumns(FAMILY, Bytes.toBytes("12"), T+3); + region.delete(d, null, false); + + Scan s = new Scan(); + s.setRaw(true); + RegionScanner scanner = region.getScanner(s); + List kvs = new ArrayList(); + scanner.next(kvs); + scanner.next(kvs); + assertEquals(8, kvs.size()); + assertTrue(kvs.get(0).isDelete()); // delete for 12 + assertArrayEquals(Bytes.toBytes("12"), kvs.get(1).getQualifier()); + assertTrue(kvs.get(2).isDelete()); // delete prefix for 123 + assertTrue(kvs.get(3).isDelete()); // delete for 123 + assertArrayEquals(Bytes.toBytes("123"), kvs.get(4).getQualifier()); + assertArrayEquals(Bytes.toBytes("1234"), kvs.get(5).getQualifier()); + assertArrayEquals(Bytes.toBytes("12345"), kvs.get(6).getQualifier()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(7).getQualifier()); + scanner.close(); + + s = new Scan(); + scanner = region.getScanner(s); + kvs = new ArrayList(); + scanner.next(kvs); + assertEquals(1, kvs.size()); + assertArrayEquals(Bytes.toBytes("125"), kvs.get(0).getQualifier()); + scanner.close(); + + region.close(); + region.getLog().closeAndDelete(); + } +} Index: src/main/java/org/apache/hadoop/hbase/client/Delete.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/Delete.java (revision 1235132) +++ src/main/java/org/apache/hadoop/hbase/client/Delete.java (working copy) @@ -202,6 +202,38 @@ } /** + * Delete all versions of all columns that match the specified qualifier prefix. + * @param family family name + * @param qualifierPrefix column qualifier prefix to match + * @return this for invocation chaining + */ + public Delete deleteColumnsByPrefix(byte[] family, byte[] qualifierPrefix) { + this.deleteColumnsByPrefix(family, qualifierPrefix, + HConstants.LATEST_TIMESTAMP); + return this; + } + + /** + * Delete all versions of all columns that match the specified qualifier prefix + * with a timestamp less than r equal to the specified timestamp. + * @param family family name + * @param qualifierPrefix column qualifier prefix to match + * @param timestamp maximum version timestamp + * @return this for invocation chaining + */ + public Delete deleteColumnsByPrefix(byte[] family, byte[] qualifierPrefix, + long timestamp) { + List list = familyMap.get(family); + if (list == null) { + list = new ArrayList(); + } + list.add(new KeyValue(this.row, family, qualifierPrefix, timestamp, + KeyValue.Type.DeleteColumnPrefix)); + familyMap.put(family, list); + return this; + } + + /** * Delete all versions of the specified column. * @param family family name * @param qualifier column qualifier Index: src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (revision 1235132) +++ src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (working copy) @@ -214,9 +214,9 @@ return MatchCode.DONE_SCAN; } - byte [] bytes = kv.getBuffer(); + final byte [] bytes = kv.getBuffer(); int offset = kv.getOffset(); - int initialOffset = offset; + final int initialOffset = offset; int keyLength = Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT); offset += KeyValue.ROW_OFFSET; Index: src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java (revision 1235132) +++ src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java (working copy) @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult; import org.apache.hadoop.hbase.util.Bytes; /** @@ -48,6 +47,11 @@ private byte deleteType = 0; private long deleteTimestamp = 0L; + private byte [] prefixBuffer = null; + private int prefixOffset = 0; + private int prefixLength = 0; + private long prefixTimestamp = 0L; + /** * Constructor for ScanDeleteTracker */ @@ -75,6 +79,15 @@ return; } + if (type == KeyValue.Type.DeleteColumnPrefix.getCode()) { + // keep prefix delete marker separate from more specific markers + // similar to how family delete markers are handled + prefixBuffer = buffer; + prefixOffset = qualifierOffset; + prefixLength = qualifierLength; + prefixTimestamp = timestamp; + return; + } if (deleteBuffer != null && type < deleteType) { // same column, so ignore less specific delete if (Bytes.equals(deleteBuffer, deleteOffset, deleteLength, @@ -109,6 +122,20 @@ return DeleteResult.FAMILY_DELETED; } + if (prefixBuffer != null) { + // check if the a prefix of the column matches the prefix delete marker + if (prefixLength <= qualifierLength + && Bytes.compareTo(prefixBuffer, prefixOffset, prefixLength, buffer, + qualifierOffset, prefixLength) == 0) { + if (timestamp <= prefixTimestamp) { + return DeleteResult.COLUMN_DELETED; + } + } else { + // past the prefix marker + prefixBuffer = null; + } + } + if (deleteBuffer != null) { int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, buffer, qualifierOffset, qualifierLength); @@ -144,7 +171,7 @@ @Override public boolean isEmpty() { - return deleteBuffer == null && familyStamp == 0; + return deleteBuffer == null && familyStamp == 0 && prefixBuffer == null; } @Override @@ -152,6 +179,7 @@ public void reset() { familyStamp = 0L; deleteBuffer = null; + prefixBuffer = null; } @Override Index: src/main/java/org/apache/hadoop/hbase/KeyValue.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/KeyValue.java (revision 1235132) +++ src/main/java/org/apache/hadoop/hbase/KeyValue.java (working copy) @@ -161,6 +161,7 @@ Delete((byte)8), DeleteColumn((byte)12), + DeleteColumnPrefix((byte)13), DeleteFamily((byte)14), // Maximum is used when searching; you look from maximum on down. @@ -2061,6 +2062,20 @@ loffset + (llength - TIMESTAMP_TYPE_SIZE)); long rtimestamp = Bytes.toLong(right, roffset + (rlength - TIMESTAMP_TYPE_SIZE)); + + // Sort column prefix delete markers before all KVs with the same column + // prefix. Make sure to keep correct order between prefix markers. + if (ltype == Type.DeleteColumnPrefix.getCode()) { + if (rtype != Type.DeleteColumnPrefix.getCode()) { + ltimestamp = Long.MAX_VALUE; + } + } + if (rtype == Type.DeleteColumnPrefix.getCode()) { + if (ltype != Type.DeleteColumnPrefix.getCode()) { + rtimestamp = Long.MAX_VALUE; + } + } + compare = compareTimestamps(ltimestamp, rtimestamp); if (compare != 0) { return compare;