.../java/org/apache/hadoop/hbase/KeyValue.java | 17 ++-------- .../java/org/apache/hadoop/hbase/KeyValueUtil.java | 36 ++++++++++++++++++++ .../apache/hadoop/hbase/codec/KeyValueCodec.java | 2 +- .../hadoop/hbase/codec/KeyValueCodecWithTags.java | 2 +- .../hadoop/hbase/regionserver/DefaultMemStore.java | 19 ++++++++--- .../hbase/regionserver/DefaultStoreFlusher.java | 2 +- .../hbase/regionserver/MemStoreSnapshot.java | 11 ++++++- .../hadoop/hbase/regionserver/TestHRegion.java | 17 ++++++++++ .../security/visibility/TestVisibilityLabels.java | 38 ++++++++++++++++++++++ 9 files changed, 122 insertions(+), 22 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index bae5fb4..26d215c 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -2498,22 +2498,11 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, * @return Created KeyValue OR if we find a length of zero, we will return null which * can be useful marking a stream as done. * @throws IOException + * @{@link Deprecated} Use {@link KeyValueUtil#iscreate(InputStream, boolean)} */ + @Deprecated public static KeyValue iscreate(final InputStream in) throws IOException { - byte [] intBytes = new byte[Bytes.SIZEOF_INT]; - int bytesRead = 0; - while (bytesRead < intBytes.length) { - int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead); - if (n < 0) { - if (bytesRead == 0) return null; // EOF at start is ok - throw new IOException("Failed read of int, read " + bytesRead + " bytes"); - } - bytesRead += n; - } - // TODO: perhaps some sanity check is needed here. - byte [] bytes = new byte[Bytes.toInt(intBytes)]; - IOUtils.readFully(in, bytes, 0, bytes.length); - return new KeyValue(bytes, 0, bytes.length); + return KeyValueUtil.iscreate(in, true); } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java index 8505fe9..adf0ad7 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hbase; +import java.io.DataInput; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -31,6 +33,7 @@ import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.IterableUtils; import org.apache.hadoop.hbase.util.SimpleMutableByteRange; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableUtils; import com.google.common.base.Function; @@ -542,6 +545,39 @@ public class KeyValueUtil { return new ArrayList(lazyList); } + /** + * Create a KeyValue reading from the raw InputStream. Named + * iscreate so doesn't clash with {@link #create(DataInput)} + * + * @param in + * @param withTags + * whether the keyvalue should include tags are not + * @return Created KeyValue OR if we find a length of zero, we will return + * null which can be useful marking a stream as done. + * @throws IOException + */ + public static KeyValue iscreate(final InputStream in, boolean withTags) throws IOException { + byte[] intBytes = new byte[Bytes.SIZEOF_INT]; + int bytesRead = 0; + while (bytesRead < intBytes.length) { + int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead); + if (n < 0) { + if (bytesRead == 0) + return null; // EOF at start is ok + throw new IOException("Failed read of int, read " + bytesRead + " bytes"); + } + bytesRead += n; + } + // TODO: perhaps some sanity check is needed here. + byte[] bytes = new byte[Bytes.toInt(intBytes)]; + IOUtils.readFully(in, bytes, 0, bytes.length); + if (withTags) { + return new KeyValue(bytes, 0, bytes.length); + } else { + return new NoTagsKeyValue(bytes, 0, bytes.length); + } + } + public static void oswrite(final Cell cell, final OutputStream out, final boolean withTags) throws IOException { if (cell instanceof KeyValue) { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java index cfe9742..07fd838 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java @@ -65,7 +65,7 @@ public class KeyValueCodec implements Codec { } protected Cell parseCell() throws IOException { - return KeyValue.iscreate(in); + return KeyValueUtil.iscreate(in, false); } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java index 02158f4..5d34a46 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java @@ -71,7 +71,7 @@ public class KeyValueCodecWithTags implements Codec { } protected Cell parseCell() throws IOException { - return KeyValue.iscreate(in); + return KeyValueUtil.iscreate(in, true); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index 3da0c0b..a0766ef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -99,6 +99,7 @@ public class DefaultMemStore implements MemStore { volatile MemStoreLAB allocator; volatile MemStoreLAB snapshotAllocator; volatile long snapshotId; + volatile boolean tagsPresent; /** * Default constructor. Used for tests. @@ -170,8 +171,11 @@ public class DefaultMemStore implements MemStore { timeOfOldestEdit = Long.MAX_VALUE; } } - return new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize, - this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator)); + MemStoreSnapshot memStoreSnapshot = new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize, + this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator), + this.tagsPresent); + this.tagsPresent = false; + return memStoreSnapshot; } /** @@ -233,6 +237,13 @@ public class DefaultMemStore implements MemStore { private boolean addToCellSet(Cell e) { boolean b = this.cellSet.add(e); + // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. + // When we use ACL CP or Visibility CP which deals with Tags during + // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not + // parse the byte[] to identify the tags length. + if(e.getTagsLength() > 0) { + tagsPresent = true; + } setOldestEditTimeToNow(); return b; } @@ -1005,8 +1016,8 @@ public class DefaultMemStore implements MemStore { } } - public final static long FIXED_OVERHEAD = ClassSize.align( - ClassSize.OBJECT + (9 * ClassSize.REFERENCE) + (3 * Bytes.SIZEOF_LONG)); + public final static long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT + + (9 * ClassSize.REFERENCE) + (3 * Bytes.SIZEOF_LONG) + Bytes.SIZEOF_BOOLEAN); public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + ClassSize.ATOMIC_LONG + (2 * ClassSize.TIMERANGE_TRACKER) + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java index 73b8cb9..474a44a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java @@ -64,7 +64,7 @@ public class DefaultStoreFlusher extends StoreFlusher { status.setStatus("Flushing " + store + ": creating writer"); // Write the map out to the disk writer = store.createWriterInTmp( - cellsCount, store.getFamily().getCompression(), false, true, true); + cellsCount, store.getFamily().getCompression(), false, true, snapshot.isTagsPresent()); writer.setTimeRangeTracker(snapshot.getTimeRangeTracker()); IOException e = null; try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java index 619cff5..be853c5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java @@ -32,14 +32,16 @@ public class MemStoreSnapshot { private final long size; private final TimeRangeTracker timeRangeTracker; private final KeyValueScanner scanner; + private final boolean tagsPresent; public MemStoreSnapshot(long id, int cellsCount, long size, TimeRangeTracker timeRangeTracker, - KeyValueScanner scanner) { + KeyValueScanner scanner, boolean tagsPresent) { this.id = id; this.cellsCount = cellsCount; this.size = size; this.timeRangeTracker = timeRangeTracker; this.scanner = scanner; + this.tagsPresent = tagsPresent; } /** @@ -76,4 +78,11 @@ public class MemStoreSnapshot { public KeyValueScanner getScanner() { return this.scanner; } + + /** + * @return true if tags are present in this snapshot + */ + public boolean isTagsPresent() { + return this.tagsPresent; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 32878d6..b61416c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -5796,6 +5796,23 @@ public class TestHRegion { } @Test + public void testFlushedFileWithNoTags() throws Exception { + String method = "testFlushedFileWithNoTags"; + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(fam1)); + region = initHRegion(Bytes.toBytes(method), method, TEST_UTIL.getConfiguration(), fam1); + Put put = new Put(Bytes.toBytes("a-b-0-0")); + put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); + region.put(put); + region.flush(true); + Store store = region.getStore(fam1); + Collection storefiles = store.getStorefiles(); + for (StoreFile sf : storefiles) { + assertFalse("Tags should not be present " + ,sf.getReader().getHFileReader().getFileContext().isIncludesTags()); + } + } + @Test @SuppressWarnings("unchecked") public void testOpenRegionWrittenToWALForLogReplay() throws Exception { // similar to the above test but with distributed log replay diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java index 85e947b..185893a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java @@ -30,6 +30,7 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -56,8 +58,11 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResul import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; @@ -805,6 +810,39 @@ public abstract class TestVisibilityLabels { } } + @Test + public void testFlushedFileWithVisibilityTags() throws Exception { + final byte[] qual2 = Bytes.toBytes("qual2"); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTableDescriptor desc = new HTableDescriptor(tableName); + HColumnDescriptor col = new HColumnDescriptor(fam); + desc.addFamily(col); + TEST_UTIL.getHBaseAdmin().createTable(desc); + try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { + Put p1 = new Put(row1); + p1.add(fam, qual, value); + p1.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + + Put p2 = new Put(row1); + p2.add(fam, qual2, value); + p2.setCellVisibility(new CellVisibility(SECRET)); + + RowMutations rm = new RowMutations(row1); + rm.add(p1); + rm.add(p2); + + table.mutateRow(rm); + } + TEST_UTIL.getHBaseAdmin().flush(tableName); + List regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); + Store store = regions.get(0).getStore(fam); + Collection storefiles = store.getStorefiles(); + assertTrue(storefiles.size() > 0); + for (StoreFile storeFile : storefiles) { + assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags()); + } + } + static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps) throws Exception { List puts = new ArrayList();