diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java index 559c593..2bff3ab 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java @@ -112,7 +112,9 @@ class AccessControlFilter extends FilterBase { cell.getQualifierLength()); currentVersions = 0; } - currentVersions++; + if (!CellUtil.isDelete(cell)) { + currentVersions++; + } if (currentVersions > familyMaxVersions) { return ReturnCode.SKIP; } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index f21d8e2..690b40a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -1516,14 +1516,24 @@ public class AccessController extends BaseMasterAndRegionObserver User user = getActiveUser(); RegionCoprocessorEnvironment env = c.getEnvironment(); Map> families = null; + boolean isRaw = false; + int requestedVersion = 0; switch (opType) { case GET: - case EXISTS: - families = ((Get)query).getFamilyMap(); + case EXISTS: { + Get g = (Get) query; + families = g.getFamilyMap(); + isRaw = false; + requestedVersion = g.getMaxVersions(); break; - case SCAN: - families = ((Scan)query).getFamilyMap(); + } + case SCAN: { + Scan s = (Scan) query; + families = s.getFamilyMap(); + isRaw = s.isRaw(); + requestedVersion = s.getMaxVersions(); break; + } default: throw new RuntimeException("Unhandled operation " + opType); } @@ -1532,7 +1542,8 @@ public class AccessController extends BaseMasterAndRegionObserver TableName table = getTableName(region); Map cfVsMaxVersions = Maps.newHashMap(); for (HColumnDescriptor hcd : region.getTableDesc().getFamilies()) { - cfVsMaxVersions.put(new SimpleMutableByteRange(hcd.getName()), hcd.getMaxVersions()); + int version = isRaw ? requestedVersion : hcd.getMaxVersions(); + cfVsMaxVersions.put(new SimpleMutableByteRange(hcd.getName()), version); } if (!authResult.isAllowed()) { if (!cellFeaturesEnabled || compatibleEarlyTermination) { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java index f1aec09..fa71373 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java @@ -331,7 +331,7 @@ public class DefaultVisibilityLabelServiceImpl implements VisibilityLabelService if (user != null && user.length > 0) { s.addColumn(LABELS_TABLE_FAMILY, user); } - Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion, + Filter filter = VisibilityUtils.createVisibilityLabelFilter(s, this.labelsRegion, new Authorizations(SYSTEM_LABEL)); s.setFilter(filter); ArrayList auths = new ArrayList(); @@ -368,7 +368,7 @@ public class DefaultVisibilityLabelServiceImpl implements VisibilityLabelService s.addColumn(LABELS_TABLE_FAMILY, Bytes.toBytes(AuthUtil.toGroupEntry(group))); } } - Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion, + Filter filter = VisibilityUtils.createVisibilityLabelFilter(s, this.labelsRegion, new Authorizations(SYSTEM_LABEL)); s.setFilter(filter); Set auths = new HashSet(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index 3fa3fa3..cd71e29 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -573,8 +573,7 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements return s; } } - - Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(region, + Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(scan, region, authorizations); if (visibilityLabelFilter != null) { Filter filter = scan.getFilter(); @@ -679,7 +678,7 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements return; } } - Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(e.getEnvironment() + Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(get, e.getEnvironment() .getRegion(), authorizations); if (visibilityLabelFilter != null) { Filter filter = get.getFilter(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java index ef7e88a..fec049f 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.security.visibility; import java.io.IOException; import java.util.Map; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -33,14 +32,12 @@ import org.apache.hadoop.hbase.util.SimpleMutableByteRange; */ @InterfaceAudience.Private class VisibilityLabelFilter extends FilterBase { - private final VisibilityExpEvaluator expEvaluator; private final Map cfVsMaxVersions; private final ByteRange curFamily; private final ByteRange curQualifier; private int curFamilyMaxVersions; private int curQualMetVersions; - public VisibilityLabelFilter(VisibilityExpEvaluator expEvaluator, Map cfVsMaxVersions) { this.expEvaluator = expEvaluator; @@ -75,11 +72,12 @@ class VisibilityLabelFilter extends FilterBase { cell.getQualifierLength()); curQualMetVersions = 0; } - curQualMetVersions++; + if (!CellUtil.isDelete(cell)) { + curQualMetVersions++; + } if (curQualMetVersions > curFamilyMaxVersions) { return ReturnCode.SKIP; } - return this.expEvaluator.evaluate(cell) ? ReturnCode.INCLUDE : ReturnCode.SKIP; } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java index 1db506d..f0d676e 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java @@ -43,6 +43,8 @@ import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.TagUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.io.util.StreamUtils; @@ -147,7 +149,7 @@ public class VisibilityUtils { * @return User auth details * @throws DeserializationException */ - public static MultiUserAuthorizations readUserAuthsFromZKData(byte[] data) + public static MultiUserAuthorizations readUserAuthsFromZKData(byte[] data) throws DeserializationException { if (ProtobufUtil.isPBMagicPrefix(data)) { int pblen = ProtobufUtil.lengthOfPBMagic(); @@ -263,20 +265,29 @@ public class VisibilityUtils { } return false; } + public static Filter createVisibilityLabelFilter(Get get, Region region, Authorizations authorizations) + throws IOException { + return createVisibilityLabelFilter(region, authorizations, get.getMaxVersions(), false); + } - public static Filter createVisibilityLabelFilter(Region region, Authorizations authorizations) - throws IOException { - Map cfVsMaxVersions = new HashMap(); + public static Filter createVisibilityLabelFilter(Scan scan, Region region, Authorizations authorizations) + throws IOException { + return createVisibilityLabelFilter(region, authorizations, scan.getMaxVersions(), scan.isRaw()); + } + + private static Filter createVisibilityLabelFilter(Region region, Authorizations authorizations, + int requestedMaxVersion, boolean isRaw) throws IOException { + Map cfVsMaxVersions = new HashMap<>(); for (HColumnDescriptor hcd : region.getTableDesc().getFamilies()) { - cfVsMaxVersions.put(new SimpleMutableByteRange(hcd.getName()), hcd.getMaxVersions()); + int maxVersions = isRaw ? requestedMaxVersion : hcd.getMaxVersions(); + cfVsMaxVersions.put(new SimpleMutableByteRange(hcd.getName()), maxVersions); } VisibilityLabelService vls = VisibilityLabelServiceManager.getInstance() - .getVisibilityLabelService(); + .getVisibilityLabelService(); Filter visibilityLabelFilter = new VisibilityLabelFilter( - vls.getVisibilityExpEvaluator(authorizations), cfVsMaxVersions); + vls.getVisibilityExpEvaluator(authorizations), cfVsMaxVersions); return visibilityLabelFilter; } - /** * @return User who called RPC method. For non-RPC handling, falls back to system user * @throws IOException When there is IOE in getting the system user (During non-RPC handling). diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java index cf01463..abce341 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java @@ -18,21 +18,27 @@ package org.apache.hadoop.hbase.security.access; +import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; - import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; import java.util.UUID; - import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -43,6 +49,7 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; +import static org.junit.Assert.assertNotNull; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; @@ -54,12 +61,13 @@ import org.junit.rules.TestName; public class TestAccessControlFilter extends SecureTestUtil { @Rule public TestName name = new TestName(); private static HBaseTestingUtility TEST_UTIL; - + private static HBaseTestingUtility NORMAL_UTIL; private static User READER; private static User LIMITED; private static User DENIED; private static TableName TABLE; + private static byte[] ROW = Bytes.toBytes("row"); private static byte[] FAMILY = Bytes.toBytes("f1"); private static byte[] PRIVATE_COL = Bytes.toBytes("private"); private static byte[] PUBLIC_COL = Bytes.toBytes("public"); @@ -87,11 +95,14 @@ public class TestAccessControlFilter extends SecureTestUtil { READER = User.createUserForTesting(conf, "reader", new String[0]); LIMITED = User.createUserForTesting(conf, "limited", new String[0]); DENIED = User.createUserForTesting(conf, "denied", new String[0]); + NORMAL_UTIL = new HBaseTestingUtility(); + NORMAL_UTIL.startMiniCluster(); } @AfterClass public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); + NORMAL_UTIL.shutdownMiniCluster(); } @Test (timeout=180000) @@ -203,4 +214,196 @@ public class TestAccessControlFilter extends SecureTestUtil { } }); } + @Test + public void testRawScan() throws Exception { + TableName tableName = TableName.valueOf("testRawScan"); + final long now = System.currentTimeMillis(); + byte[] row1 = Bytes.toBytes("row1"); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY) + .setMaxVersions(5) + .setKeepDeletedCells(KeepDeletedCells.TRUE) + ); + createTable(TEST_UTIL, TEST_UTIL.getAdmin(), desc, null); + SecureTestUtil.grantOnTable(TEST_UTIL, LIMITED.getShortName(), + tableName, FAMILY, PUBLIC_COL, Permission.Action.READ); + try (Table t = TEST_UTIL.getConnection().getTable(desc.getTableName())) { + Put p = new Put(row1); + // all allowed + p.addColumn(FAMILY, PUBLIC_COL, now, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 1, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 2, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 3, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 4, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 5, PUBLIC_COL); + t.put(p); + // denied to raw scan + Delete d = new Delete(row1, now + 3); + t.delete(d); + d = new Delete(row1); + // allowed + d.addColumns(FAMILY, PUBLIC_COL, now + 2); + t.delete(d); + } + final Get g = new Get(row1); + final Scan s = new Scan(); + s.setRaw(true); + // all versions + LIMITED.runAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + s.setMaxVersions(); + g.setMaxVersions(); + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + try (Connection connection = ConnectionFactory.createConnection(conf)) { + try (Table t = connection.getTable(TABLE); + ResultScanner scanner = t.getScanner(s)) { + Result result = scanner.next(); + Cell[] res = result.rawCells(); + assertNotNull(res); + assertEquals(7, res.length); + assertEquals(now + 5, res[0].getTimestamp()); + assertEquals(now + 4, res[1].getTimestamp()); + assertEquals(now + 3, res[2].getTimestamp()); + assertEquals(true, CellUtil.isDelete(res[3])); + assertEquals(now + 2, res[4].getTimestamp()); + assertEquals(now + 1, res[5].getTimestamp()); + assertEquals(now, res[6].getTimestamp()); + } + try (Table t = connection.getTable(TABLE)) { + Result result = t.get(g); + Cell[] res = result.rawCells(); + assertNotNull(res); + assertEquals(2, res.length); + assertEquals(now + 5, res[0].getTimestamp()); + assertEquals(now + 4, res[1].getTimestamp()); + } + } + return null; + } + }); + // one version + LIMITED.runAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + s.setMaxVersions(1); + g.setMaxVersions(1); + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + try (Connection connection = ConnectionFactory.createConnection(conf)) { + try (Table t = connection.getTable(TABLE); + ResultScanner scanner = t.getScanner(s)) { + Result result = scanner.next(); + Cell[] res = result.rawCells(); + assertEquals(1, res.length); + assertEquals(now + 5, res[0].getTimestamp()); + } + try (Table t = connection.getTable(TABLE)) { + Result result = t.get(g); + Cell[] res = result.rawCells(); + assertNotNull(res); + assertEquals(1, res.length); + assertEquals(now + 5, res[0].getTimestamp()); + } + } + return null; + } + }); + deleteTable(TEST_UTIL, TEST_UTIL.getAdmin(), desc.getTableName()); + } + + private static void createMultiVersionTable(HBaseTestingUtility util, final TableName tableName, + final int versions, final long now) throws IOException { + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY) + .setMaxVersions(versions) + .setKeepDeletedCells(KeepDeletedCells.TRUE) + ); + util.getAdmin().createTable(desc); + try (Table t = util.getConnection().getTable(desc.getTableName())) { + Put p = new Put(ROW); + p.addColumn(FAMILY, PUBLIC_COL, now, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 1, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 2, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 3, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 4, PUBLIC_COL); + p.addColumn(FAMILY, PUBLIC_COL, now + 5, PUBLIC_COL); + t.put(p); + Delete d = new Delete(ROW, now + 3); + t.delete(d); + d = new Delete(ROW); + d.addColumns(FAMILY, PUBLIC_COL, now + 2); + t.delete(d); + } + } + private static Result getResult(HBaseTestingUtility util, final TableName tableName, final Get g)throws IOException { + try (Table t = util.getConnection().getTable(tableName)) { + Result r = t.get(g); + assertNotNull(r); + return r; + } + } + private static Result getResult(HBaseTestingUtility util, final TableName tableName, final Scan s) throws IOException { + try (Table t = util.getConnection().getTable(tableName); + ResultScanner scanner = t.getScanner(s)) { + Result r = scanner.next(); + assertNotNull(r); + return r; + } + } + @Test + public void testRawScanWithAndWithoutSecurity() throws Exception { + final TableName tableName = TableName.valueOf("testRawScanWoSecurity"); + long now = System.currentTimeMillis(); + createMultiVersionTable(NORMAL_UTIL, tableName, 5, now); + createMultiVersionTable(TEST_UTIL, tableName, 5, now); + final Scan s = new Scan(); + s.setRaw(true); + + // all versions + s.setMaxVersions(); + Result r1 = getResult(NORMAL_UTIL, tableName, s); + Result r2 = LIMITED.runAs(new PrivilegedExceptionAction() { + @Override + public Result run() throws Exception { + return getResult(TEST_UTIL, tableName, s); + } + }); + Result.compareResults(r1, r2); + + // one version + s.setMaxVersions(1); + Result r3 = getResult(NORMAL_UTIL, tableName, s); + Result r4 = LIMITED.runAs(new PrivilegedExceptionAction() { + @Override + public Result run() throws Exception { + return getResult(TEST_UTIL, tableName, s); + } + }); + Result.compareResults(r3, r4); + final Get g = new Get(ROW); + // one version + g.setMaxVersions(); + Result r5 = getResult(NORMAL_UTIL, tableName, g); + Result r6 = LIMITED.runAs(new PrivilegedExceptionAction() { + @Override + public Result run() throws Exception { + return getResult(TEST_UTIL, tableName, g); + } + }); + Result.compareResults(r5, r6); + + // one version + g.setMaxVersions(1); + Result r7 = getResult(NORMAL_UTIL, tableName, g); + Result r8 = LIMITED.runAs(new PrivilegedExceptionAction() { + @Override + public Result run() throws Exception { + return getResult(TEST_UTIL, tableName, g); + } + }); + Result.compareResults(r7, r8); + deleteTable(TEST_UTIL, TEST_UTIL.getAdmin(), tableName); + NORMAL_UTIL.getAdmin().disableTable(tableName); + NORMAL_UTIL.getAdmin().deleteTable(tableName); + } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java index 63c08a2..5eced0d 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java @@ -19,16 +19,12 @@ package org.apache.hadoop.hbase.security.visibility; import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; - import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HConstants; @@ -54,14 +50,27 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; - import com.google.protobuf.ByteString; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeepDeletedCells; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import static org.apache.hadoop.hbase.security.visibility.TestVisibilityLabels.TEST_UTIL; +import org.junit.AfterClass; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; @Category({SecurityTests.class, MediumTests.class}) public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibilityLabels { private static final Log LOG = LogFactory.getLog( TestVisibilityLabelsWithDefaultVisLabelService.class); - + private static final HBaseTestingUtility NORMAL_UTIL = new HBaseTestingUtility(); @BeforeClass public static void setupBeforeClass() throws Exception { // setup configuration @@ -78,8 +87,13 @@ public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibili // Wait for the labels table to become available TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000); addLabels(); + NORMAL_UTIL.startMiniCluster(2); + NORMAL_UTIL.waitUntilNoRegionsInTransition(); + } + @AfterClass + public static void tearDownAfterClass() throws Exception { + NORMAL_UTIL.shutdownMiniCluster(); } - @Test public void testAddLabels() throws Throwable { PrivilegedExceptionAction action = @@ -240,4 +254,158 @@ public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibili assertTrue(next.length == 1); } } + @Test + public void testRawScan() throws Exception { + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + long now = System.currentTimeMillis(); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(fam) + .setMaxVersions(5) + .setKeepDeletedCells(KeepDeletedCells.TRUE) + ); + TEST_UTIL.getAdmin().createTable(desc); + try (Table t = TEST_UTIL.getConnection().getTable(desc.getTableName())) { + Put p = new Put(row1); + p.addColumn(fam, qual, now, qual); + p.addColumn(fam, qual, now + 1, qual); + p.addColumn(fam, qual, now + 2, qual); + p.addColumn(fam, qual, now + 3, qual); + p.addColumn(fam, qual, now + 4, qual); + p.addColumn(fam, qual, now + 5, qual); + t.put(p); + Delete d = new Delete(row1, now + 3); + t.delete(d); + d = new Delete(row1); + d.addColumns(fam, qual, now + 2); + t.delete(d); + } + Scan s = new Scan(); + s.setMaxVersions(); + s.setRaw(true); + // all versions + try (Table t = TEST_UTIL.getConnection().getTable(desc.getTableName()); + ResultScanner scanner = t.getScanner(s)) { + Result result = scanner.next(); + Cell[] res = result.rawCells(); + assertEquals(8, res.length); + assertTrue(CellUtil.isDeleteFamily(res[0])); + assertEquals(now + 5, res[1].getTimestamp()); + assertEquals(now + 4, res[2].getTimestamp()); + assertEquals(now + 3, res[3].getTimestamp()); + assertTrue(CellUtil.isDelete(res[4])); + assertEquals(now + 2, res[5].getTimestamp()); + assertEquals(now + 1, res[6].getTimestamp()); + assertEquals(now, res[7].getTimestamp()); + } + // one version + s.setMaxVersions(1); + try (Table t = TEST_UTIL.getConnection().getTable(desc.getTableName()); + ResultScanner scanner = t.getScanner(s)) { + Result result = scanner.next(); + Cell[] res = result.rawCells(); + assertEquals(2, res.length); + assertTrue(CellUtil.isDeleteFamily(res[0])); + assertEquals(now + 5, res[1].getTimestamp()); + } + + Get get = new Get(row1); + + // all versions + get.setMaxVersions(); + try (Table t = TEST_UTIL.getConnection().getTable(desc.getTableName())) { + Result result = t.get(get); + Cell[] res = result.rawCells(); + assertEquals(2, res.length); + assertEquals(now + 5, res[0].getTimestamp()); + assertEquals(now + 4, res[1].getTimestamp()); + } + + // single versions + get.setMaxVersions(1); + try (Table t = TEST_UTIL.getConnection().getTable(desc.getTableName())) { + Result result = t.get(get); + Cell[] res = result.rawCells(); + assertEquals(1, res.length); + assertEquals(now + 5, res[0].getTimestamp()); + } + TEST_UTIL.getAdmin().disableTable(desc.getTableName()); + TEST_UTIL.getAdmin().deleteTable(desc.getTableName()); + } + private static void createMultiVersionTable(HBaseTestingUtility util, final TableName tableName, final long now) throws IOException { + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(fam) + .setMaxVersions(5) + .setKeepDeletedCells(KeepDeletedCells.TRUE) + ); + util.getAdmin().createTable(desc); + try (Table t = util.getConnection().getTable(desc.getTableName())) { + Put p = new Put(row1); + p.addColumn(fam, qual, now, qual); + p.addColumn(fam, qual, now + 1, qual); + p.addColumn(fam, qual, now + 2, qual); + p.addColumn(fam, qual, now + 3, qual); + p.addColumn(fam, qual, now + 4, qual); + p.addColumn(fam, qual, now + 5, qual); + t.put(p); + Delete d = new Delete(row1, now + 3); + t.delete(d); + d = new Delete(row1); + d.addColumns(fam, qual, now + 2); + t.delete(d); + } + } + private static Result getResult(HBaseTestingUtility util, final TableName tableName, final Get g)throws IOException { + try (Table t = util.getConnection().getTable(tableName)) { + Result r = t.get(g); + assertNotNull(r); + return r; + } + } + private static Result getResult(HBaseTestingUtility util, final TableName tableName, final Scan s) throws IOException { + try (Table t = util.getConnection().getTable(tableName); + ResultScanner scanner = t.getScanner(s)) { + Result r = scanner.next(); + assertNotNull(r); + return r; + } + } + + @Test + public void testRawScanWithAndWithoutVisibilityLabels() throws Exception { + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + long now = System.currentTimeMillis(); + createMultiVersionTable(NORMAL_UTIL, tableName, now); + createMultiVersionTable(TEST_UTIL, tableName, now); + Scan s = new Scan(); + s.setRaw(true); + + // all versions + s.setMaxVersions(); + Result r1 = getResult(NORMAL_UTIL, tableName, s); + Result r2 = getResult(TEST_UTIL, tableName, s); + Result.compareResults(r1, r2); + + // one version + s.setMaxVersions(1); + Result r3 = getResult(NORMAL_UTIL, tableName, s); + Result r4 = getResult(TEST_UTIL, tableName, s); + Result.compareResults(r3, r4); + + Get g = new Get(row1); + // one version + g.setMaxVersions(); + Result r5 = getResult(NORMAL_UTIL, tableName, g); + Result r6 = getResult(TEST_UTIL, tableName, g); + Result.compareResults(r5, r6); + + // one version + g.setMaxVersions(1); + Result r7 = getResult(NORMAL_UTIL, tableName, g); + Result r8 = getResult(TEST_UTIL, tableName, g); + Result.compareResults(r7, r8); + NORMAL_UTIL.getAdmin().disableTable(tableName); + NORMAL_UTIL.getAdmin().deleteTable(tableName); + TEST_UTIL.getAdmin().disableTable(tableName); + TEST_UTIL.getAdmin().deleteTable(tableName); + } }