.../org/apache/hadoop/hbase/HTableDescriptor.java | 21 +++++++ .../security/visibility/VisibilityConstants.java | 7 +++ ...IntegrationTestBigLinkedListWithVisibility.java | 1 + ...grationTestWithCellVisibilityLoadAndVerify.java | 1 + .../security/visibility/VisibilityController.java | 45 ++++++++++----- .../visibility/VisibilityLabelsManager.java | 24 ++++++++ .../TestImportTSVWithVisibilityLabels.java | 31 ++++------- .../hadoop/hbase/rest/TestScannersWithLabels.java | 1 + .../TestEnforcingScanLabelGenerator.java | 10 ++-- .../security/visibility/TestVisibilityLabels.java | 42 ++++++++++++-- .../visibility/TestVisibilityLabelsWithACL.java | 8 +++ .../TestVisibilityLabelsWithDeletes.java | 64 +++++++++++++++++++++- ...stVisibilityLabelsWithDistributedLogReplay.java | 1 - .../TestVisibilityLabelsWithSLGStack.java | 4 +- .../org/apache/hadoop/hbase/util/LoadTestTool.java | 1 + .../TestThriftHBaseServiceHandlerWithLabels.java | 43 +++------------ 16 files changed, 222 insertions(+), 82 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 99fe157..6e5245f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.visibility.VisibilityConstants; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.WritableComparable; @@ -1521,4 +1522,24 @@ public class HTableDescriptor implements WritableComparable { public void removeConfiguration(final String key) { configuration.remove(key); } + + /** + * Used with visibility expression. Setting this property to true would mean + * that for every mutation issued, the labels in the visibility expressions + * are validated against the set of labels associated with the user issuing + * the mutation. If not found then the mutation would fail. + * + * @param setCheckAuths + */ + public void setCheckAuthsForMutation(boolean setCheckAuths) { + setValue(VisibilityConstants.CHECK_AUTHS_FOR_MUTATION_KEY, Boolean.toString(setCheckAuths)); + } + + public boolean getCheckAuthsForMutation() { + byte[] value = getValue(VisibilityConstants.CHECK_AUTHS_FOR_MUTATION_KEY); + if (value != null) { + return Boolean.parseBoolean(Bytes.toString(value)); + } + return true; + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java index f98efec..be3183d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.security.visibility; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; @InterfaceAudience.Private @@ -49,4 +50,10 @@ public final class VisibilityConstants { public static final byte[] SORTED_ORDINAL_SERIALIZATION_FORMAT = new byte[] { VISIBILITY_SERIALIZATION_VERSION }; + /** + * Checks if the mutation has to be checked with the auths associated with the user + */ + public static final String CHECK_AUTHS_FOR_MUTATION = "CHECK_AUTHS_FOR_MUTATION"; + public static final ImmutableBytesWritable CHECK_AUTHS_FOR_MUTATION_KEY = + new ImmutableBytesWritable(Bytes.toBytes(CHECK_AUTHS_FOR_MUTATION)); } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java index 9748b31..51868d9 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java @@ -144,6 +144,7 @@ public class IntegrationTestBigLinkedListWithVisibility extends IntegrationTestB boolean acl) throws IOException { if (!admin.tableExists(tableName)) { HTableDescriptor htd = new HTableDescriptor(tableName); + htd.setCheckAuthsForMutation(false); HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME); if (setVersion) { family.setMaxVersions(DEFAULT_TABLES_COUNT); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java index 32cdfa2..a50b0ed 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java @@ -365,6 +365,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT // create HTableDescriptor for specified table String table = getTablename(); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); + htd.setCheckAuthsForMutation(false); htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); HBaseAdmin admin = new HBaseAdmin(getConf()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index 6920f7b..d28c379 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -114,6 +114,7 @@ import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode; import org.apache.hadoop.hbase.security.visibility.expression.Operator; import org.apache.hadoop.hbase.util.ByteRange; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.SimpleMutableByteRange; @@ -122,7 +123,6 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import com.google.common.collect.Lists; import com.google.common.collect.MapMaker; import com.google.protobuf.ByteString; -import org.apache.hadoop.hbase.util.ByteStringer; import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; import com.google.protobuf.Service; @@ -691,7 +691,13 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb return; } // TODO this can be made as a global LRU cache at HRS level? + boolean checkAuths = c.getEnvironment().getRegion().getTableDesc().getCheckAuthsForMutation(); Map> labelCache = new HashMap>(); + List auths = null; + User user = getActiveUser(); + if (checkAuths && user != null && user.getShortName() != null) { + auths = this.visibilityManager.getAuthsAsOrdinals(user.getShortName()); + } for (int i = 0; i < miniBatchOp.size(); i++) { Mutation m = miniBatchOp.getOperation(i); CellVisibility cellVisibility = null; @@ -717,7 +723,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb List visibilityTags = labelCache.get(labelsExp); if (visibilityTags == null) { try { - visibilityTags = createVisibilityTags(labelsExp, true); + visibilityTags = createVisibilityTags(labelsExp, true, auths, user.getShortName()); } catch (ParseException e) { miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, e.getMessage())); @@ -777,7 +783,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (cellVisibility != null) { String labelsExp = cellVisibility.getExpression(); try { - visibilityTags = createVisibilityTags(labelsExp, false); + visibilityTags = createVisibilityTags(labelsExp, false, null, null); } catch (ParseException e) { throw new IOException("Invalid cell visibility expression " + labelsExp, e); } catch (InvalidLabelException e) { @@ -911,8 +917,9 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb return true; } - private List createVisibilityTags(String visibilityLabelsExp, boolean addSerializationTag) - throws IOException, ParseException, InvalidLabelException { + private List createVisibilityTags(String visibilityLabelsExp, boolean addSerializationTag, + List auths, String userName) throws IOException, ParseException, + InvalidLabelException { ExpressionNode node = null; node = this.expressionParser.parse(visibilityLabelsExp); node = this.expressionExpander.expand(node); @@ -926,7 +933,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb tags.add(VisibilityUtils.VIS_SERIALIZATION_TAG); } if (node.isSingleNode()) { - getLabelOrdinals(node, labelOrdinals); + getLabelOrdinals(node, labelOrdinals, auths, userName); writeLabelOrdinalsToStream(labelOrdinals, dos); tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); baos.reset(); @@ -934,14 +941,14 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node; if (nlNode.getOperator() == Operator.OR) { for (ExpressionNode child : nlNode.getChildExps()) { - getLabelOrdinals(child, labelOrdinals); + getLabelOrdinals(child, labelOrdinals, auths, userName); writeLabelOrdinalsToStream(labelOrdinals, dos); tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); baos.reset(); labelOrdinals.clear(); } } else { - getLabelOrdinals(nlNode, labelOrdinals); + getLabelOrdinals(nlNode, labelOrdinals, auths, userName); writeLabelOrdinalsToStream(labelOrdinals, dos); tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); baos.reset(); @@ -958,8 +965,8 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } } - private void getLabelOrdinals(ExpressionNode node, List labelOrdinals) - throws IOException, InvalidLabelException { + private void getLabelOrdinals(ExpressionNode node, List labelOrdinals, + List auths, String userName) throws IOException, InvalidLabelException { if (node.isSingleNode()) { String identifier = null; int labelOrdinal = 0; @@ -970,12 +977,14 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb LOG.trace("The identifier is "+identifier); } labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier); + checkAuths(auths, userName, labelOrdinal, identifier); } else { // This is a NOT node. LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node) .getChildExps().get(0); identifier = lNode.getIdentifier(); labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier); + checkAuths(auths, userName, labelOrdinal, identifier); labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal. } if (labelOrdinal == 0) { @@ -985,11 +994,21 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } else { List childExps = ((NonLeafExpressionNode) node).getChildExps(); for (ExpressionNode child : childExps) { - getLabelOrdinals(child, labelOrdinals); + getLabelOrdinals(child, labelOrdinals, auths, userName); } } } - + + private void checkAuths(List auths, String userName, int labelOrdinal, String identifier) + throws InvalidLabelException, AccessDeniedException { + if (auths != null) { + if (!auths.contains(labelOrdinal)) { + throw new AccessDeniedException("Visibility label " + identifier + + " not associated with user " + userName); + } + } + } + @Override public RegionScanner preScannerOpen(ObserverContext e, Scan scan, RegionScanner s) throws IOException { @@ -1242,7 +1261,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } } try { - tags.addAll(createVisibilityTags(cellVisibility.getExpression(), true)); + tags.addAll(createVisibilityTags(cellVisibility.getExpression(), true, null, null)); } catch (ParseException e) { throw new IOException(e); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsManager.java index 7f1f278..773a9dd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsManager.java @@ -48,6 +48,7 @@ public class VisibilityLabelsManager { private static final Log LOG = LogFactory.getLog(VisibilityLabelsManager.class); private static final List EMPTY_LIST = new ArrayList(0); + private static final List EMPTY_INT_LIST = new ArrayList(0); private static VisibilityLabelsManager instance; private ZKVisibilityLabelWatcher zkVisibilityWatcher; @@ -173,6 +174,29 @@ public class VisibilityLabelsManager { } /** + * Returns the list of ordinals of authentications associated with the user + * + * @param user + * @return the list of ordinals + */ + public List getAuthsAsOrdinals(String user) { + List auths = EMPTY_INT_LIST; + this.lock.readLock().lock(); + try { + Set authOrdinals = userAuths.get(user); + if (authOrdinals != null) { + auths = new ArrayList(authOrdinals.size()); + for (Integer authOrdinal : authOrdinals) { + auths.add(authOrdinal); + } + } + } finally { + this.lock.readLock().unlock(); + } + return auths; + } + + /** * Writes the labels data to zookeeper node. * @param data * @param labelsOrUserAuths true for writing labels and false for user auths. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java index 0c483aa..68421b5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java @@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -48,7 +47,6 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.visibility.Authorizations; import org.apache.hadoop.hbase.security.visibility.CellVisibility; @@ -91,7 +89,6 @@ public class TestImportTSVWithVisibilityLabels implements Configurable { private final static String PRIVATE = "private"; private final static String CONFIDENTIAL = "confidential"; private final static String SECRET = "secret"; - private static User SUPERUSER; private static Configuration conf; @Override @@ -107,11 +104,10 @@ public class TestImportTSVWithVisibilityLabels implements Configurable { @BeforeClass public static void provisionCluster() throws Exception { conf = util.getConfiguration(); - SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); - conf.set("hbase.superuser", "admin,"+User.getCurrent().getName()); conf.setInt("hfile.format.version", 3); conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName()); conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName()); + conf.set("hbase.superuser", User.getCurrent().getName()); conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, ScanLabelGenerator.class); util.startMiniCluster(); @@ -123,22 +119,15 @@ public class TestImportTSVWithVisibilityLabels implements Configurable { } private static void createLabels() throws IOException, InterruptedException { - PrivilegedExceptionAction action = - new PrivilegedExceptionAction() { - @Override - public VisibilityLabelsResponse run() throws Exception { - String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE }; - try { - VisibilityClient.addLabels(conf, labels); - LOG.info("Added labels "); - } catch (Throwable t) { - LOG.error("Error in adding labels" , t); - throw new IOException(t); - } - return null; - } - }; - SUPERUSER.runAs(action); + String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE }; + try { + VisibilityClient.addLabels(conf, labels); + VisibilityClient.setAuths(util.getConfiguration(), labels, User.getCurrent().getName()); + LOG.info("Added labels"); + } catch (Throwable t) { + LOG.error("Error in adding labels" , t); + throw new IOException(t); + } } @AfterClass diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java index e6845f7..72111e4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithLabels.java @@ -149,6 +149,7 @@ public class TestScannersWithLabels { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE)); htd.addFamily(new HColumnDescriptor(CFA)); htd.addFamily(new HColumnDescriptor(CFB)); + htd.setCheckAuthsForMutation(false); admin.createTable(htd); insertData(TABLE, COLUMN_1, 1.0); insertData(TABLE, COLUMN_2, 0.5); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestEnforcingScanLabelGenerator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestEnforcingScanLabelGenerator.java index ac418e3..608dce0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestEnforcingScanLabelGenerator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestEnforcingScanLabelGenerator.java @@ -18,7 +18,8 @@ package org.apache.hadoop.hbase.security.visibility; import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; -import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.security.PrivilegedExceptionAction; @@ -26,15 +27,15 @@ import java.security.PrivilegedExceptionAction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; - import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; @@ -99,7 +100,8 @@ public class TestEnforcingScanLabelGenerator { SUPERUSER.runAs(new PrivilegedExceptionAction() { public Void run() throws Exception { - HTable table = TEST_UTIL.createTable(tableName, CF); + HTable table = TestVisibilityLabels.createTable(new HTableDescriptor(tableName), CF, + TEST_UTIL.getConfiguration(), TEST_UTIL); try { Put put = new Put(ROW_1); put.add(CF, Q1, HConstants.LATEST_TIMESTAMP, value); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java index 06f52e7..96f67b7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java @@ -632,7 +632,8 @@ public class TestVisibilityLabels { TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); HTable table = null; try { - table = TEST_UTIL.createTable(tableName, fam); + table = createTable(new HTableDescriptor(tableName), fam, TEST_UTIL.getConfiguration(), + TEST_UTIL); byte[] row1 = Bytes.toBytes("row1"); Put put = new Put(row1); put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value); @@ -664,7 +665,8 @@ public class TestVisibilityLabels { TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); HTable table = null; try { - table = TEST_UTIL.createTable(tableName, fam); + table = createTable(new HTableDescriptor(tableName), fam, TEST_UTIL.getConfiguration(), + TEST_UTIL); byte[] row1 = Bytes.toBytes("row1"); byte[] val = Bytes.toBytes(1L); Put put = new Put(row1); @@ -696,7 +698,8 @@ public class TestVisibilityLabels { TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); HTable table = null; try { - table = TEST_UTIL.createTable(tableName, fam); + table = createTable(new HTableDescriptor(tableName), fam, TEST_UTIL.getConfiguration(), + TEST_UTIL); byte[] row1 = Bytes.toBytes("row1"); byte[] val = Bytes.toBytes("a"); Put put = new Put(row1); @@ -778,6 +781,7 @@ public class TestVisibilityLabels { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor col = new HColumnDescriptor(fam);// Default max versions is 1. desc.addFamily(col); + desc.setCheckAuthsForMutation(false); col = new HColumnDescriptor(fam2); col.setMaxVersions(5); desc.addFamily(col); @@ -869,6 +873,7 @@ public class TestVisibilityLabels { final byte[] qual2 = Bytes.toBytes("qual2"); TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); HTableDescriptor desc = new HTableDescriptor(tableName); + desc.setCheckAuthsForMutation(false); HColumnDescriptor col = new HColumnDescriptor(fam); desc.addFamily(col); TEST_UTIL.getHBaseAdmin().createTable(desc); @@ -907,7 +912,8 @@ public class TestVisibilityLabels { throws Exception { HTable table = null; try { - table = TEST_UTIL.createTable(tableName, fam); + table = createTable(new HTableDescriptor(tableName), fam, TEST_UTIL.getConfiguration(), + TEST_UTIL); int i = 1; List puts = new ArrayList(); for (String labelExp : labelExps) { @@ -941,4 +947,32 @@ public class TestVisibilityLabels { }; SUPERUSER.runAs(action); } + + /** + * Create a table. + * + * @param htd + * @param families + * @param c + * Configuration to use + * @return An HTable instance for the created table. + * @throws IOException + */ + public static HTable createTable(HTableDescriptor htd, byte[] families, Configuration c, + HBaseTestingUtility util) throws IOException { + HColumnDescriptor hcd = new HColumnDescriptor(families); + // Disable blooms (they are on by default as of 0.95) but we disable them + // here because + // tests have hard coded counts of what to expect in block cache, etc., and + // blooms being + // on is interfering. + hcd.setBloomFilterType(BloomType.NONE); + htd.setCheckAuthsForMutation(false); + htd.addFamily(hcd); + util.getHBaseAdmin().createTable(htd); + // HBaseAdmin only waits for regions to appear in hbase:meta we should wait + // until they are assigned + util.waitUntilAllRegionsAssigned(htd.getTableName()); + return new HTable(c, htd.getTableName()); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithACL.java index 15b3136..d5b5ee7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithACL.java @@ -111,8 +111,10 @@ public class TestVisibilityLabelsWithACL { @Test public void testScanForUserWithFewerLabelAuthsThanLabelsInScanAuthorizations() throws Throwable { String[] auths = { SECRET }; + String[] labels = {SECRET, CONFIDENTIAL, PRIVATE}; String user = "user2"; VisibilityClient.setAuths(conf, auths, user); + VisibilityClient.setAuths(conf, labels, User.getCurrent().getName()); TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL + "&!" + PRIVATE, SECRET + "&!" + PRIVATE); @@ -142,8 +144,10 @@ public class TestVisibilityLabelsWithACL { @Test public void testScanForSuperUserWithFewerLabelAuths() throws Throwable { String[] auths = { SECRET }; + String[] labels = {SECRET, CONFIDENTIAL, PRIVATE}; String user = "admin"; VisibilityClient.setAuths(conf, auths, user); + VisibilityClient.setAuths(conf, labels, User.getCurrent().getName()); TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL + "&!" + PRIVATE, SECRET + "&!" + PRIVATE); @@ -168,8 +172,10 @@ public class TestVisibilityLabelsWithACL { @Test public void testGetForSuperUserWithFewerLabelAuths() throws Throwable { String[] auths = { SECRET }; + String[] labels = {SECRET, CONFIDENTIAL, PRIVATE}; String user = "admin"; VisibilityClient.setAuths(conf, auths, user); + VisibilityClient.setAuths(conf, labels, User.getCurrent().getName()); TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL + "&!" + PRIVATE, SECRET + "&!" + PRIVATE); @@ -194,8 +200,10 @@ public class TestVisibilityLabelsWithACL { public void testVisibilityLabelsForUserWithNoAuths() throws Throwable { String user = "admin"; String[] auths = { SECRET }; + String[] labels = {SECRET, CONFIDENTIAL, PRIVATE}; VisibilityClient.clearAuths(conf, auths, user); // Removing all auths if any. VisibilityClient.setAuths(conf, auths, "user1"); + VisibilityClient.setAuths(conf, labels, User.getCurrent().getName()); TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET); SecureTestUtil.grantOnTable(TEST_UTIL, NORMAL_USER1.getShortName(), tableName, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java index d3df952..f988376 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java @@ -525,6 +525,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -598,6 +599,54 @@ public class TestVisibilityLabelsWithDeletes { } @Test + public void testVerifyAccessDeniedForInvalidUserAuths() throws Exception { + PrivilegedExceptionAction action = new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { PRIVATE, SECRET, TOPSECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(true); + hBaseAdmin.createTable(desc); + HTable table = null; + try { + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + HTable table = null; + try { + table = new HTable(conf, TEST_NAME.getMethodName()); + Put p = new Put(row1); + p.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET + ")")); + p.add(fam, qual, 125l, value); + table.put(p); + } catch (Throwable t) { + assertTrue(t.getMessage().contains("AccessDeniedException")); + } finally { + table.close(); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + } catch (Exception e) { + throw new IOException(e); + } + } + + @Test public void testVisibilityLabelsWithDeleteColumnsWithPutsReAppearing() throws Exception { TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); HTable table = null; @@ -607,6 +656,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -689,6 +739,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -745,6 +796,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -934,6 +986,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -976,6 +1029,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -1011,6 +1065,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -1461,6 +1516,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -1514,6 +1570,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -2923,6 +2980,7 @@ public class TestVisibilityLabelsWithDeletes { colDesc.setMaxVersions(5); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(colDesc); + desc.setCheckAuthsForMutation(false); hBaseAdmin.createTable(desc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("row1")); @@ -2978,7 +3036,8 @@ public class TestVisibilityLabelsWithDeletes { public static HTable createTableAndWriteDataWithLabels(TableName tableName, String... labelExps) throws Exception { HTable table = null; - table = TEST_UTIL.createTable(tableName, fam); + table = TestVisibilityLabels.createTable(new HTableDescriptor(tableName), fam, + TEST_UTIL.getConfiguration(), TEST_UTIL); int i = 1; List puts = new ArrayList(); for (String labelExp : labelExps) { @@ -2996,7 +3055,8 @@ public class TestVisibilityLabelsWithDeletes { public static HTable createTableAndWriteDataWithLabels(TableName tableName, long[] timestamp, String... labelExps) throws Exception { HTable table = null; - table = TEST_UTIL.createTable(tableName, fam); + table = TestVisibilityLabels.createTable(new HTableDescriptor(tableName), fam, + TEST_UTIL.getConfiguration(), TEST_UTIL); int i = 1; List puts = new ArrayList(); for (String labelExp : labelExps) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java index 6c7fea5..ae8d741 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java @@ -22,7 +22,6 @@ import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LA import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.security.User; -import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithSLGStack.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithSLGStack.java index 4461519..0f80daf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithSLGStack.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithSLGStack.java @@ -27,6 +27,7 @@ import java.security.PrivilegedExceptionAction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HTable; @@ -85,7 +86,8 @@ public class TestVisibilityLabelsWithSLGStack { TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); HTable table = null; try { - table = TEST_UTIL.createTable(tableName, CF); + table = TestVisibilityLabels.createTable(new HTableDescriptor(tableName), CF, + TEST_UTIL.getConfiguration(), TEST_UTIL); Put put = new Put(ROW_1); put.add(CF, Q1, HConstants.LATEST_TIMESTAMP, value); put.setCellVisibility(new CellVisibility(SECRET)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java index ce3da34..ab46ace 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java @@ -245,6 +245,7 @@ public class LoadTestTool extends AbstractHBaseTool { byte[][] columnFamilies) throws IOException { HBaseAdmin admin = new HBaseAdmin(conf); HTableDescriptor tableDesc = admin.getTableDescriptor(tableName); + tableDesc.setCheckAuthsForMutation(false); LOG.info("Disabling table " + tableName); admin.disableTable(tableName); for (byte[] cf : columnFamilies) { diff --git a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandlerWithLabels.java b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandlerWithLabels.java index 29cb4ce..fb6c4d9 100644 --- a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandlerWithLabels.java +++ b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandlerWithLabels.java @@ -18,55 +18,25 @@ package org.apache.hadoop.hbase.thrift2; import static java.nio.ByteBuffer.wrap; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import java.io.IOException; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; +import java.util.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MediumTests; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator; -import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator; -import org.apache.hadoop.hbase.security.visibility.VisibilityClient; -import org.apache.hadoop.hbase.security.visibility.VisibilityConstants; -import org.apache.hadoop.hbase.security.visibility.VisibilityController; -import org.apache.hadoop.hbase.security.visibility.VisibilityUtils; -import org.apache.hadoop.hbase.thrift2.generated.TAppend; -import org.apache.hadoop.hbase.thrift2.generated.TAuthorization; -import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility; -import org.apache.hadoop.hbase.thrift2.generated.TColumn; -import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement; -import org.apache.hadoop.hbase.thrift2.generated.TColumnValue; -import org.apache.hadoop.hbase.thrift2.generated.TGet; -import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument; -import org.apache.hadoop.hbase.thrift2.generated.TIncrement; -import org.apache.hadoop.hbase.thrift2.generated.TPut; -import org.apache.hadoop.hbase.thrift2.generated.TResult; -import org.apache.hadoop.hbase.thrift2.generated.TScan; +import org.apache.hadoop.hbase.security.visibility.*; +import org.apache.hadoop.hbase.thrift2.generated.*; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.*; import org.junit.experimental.categories.Category; @Category(MediumTests.class) @@ -139,6 +109,7 @@ public static void beforeClass() throws Exception { HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); HTableDescriptor tableDescriptor = new HTableDescriptor( TableName.valueOf(tableAname)); + tableDescriptor.setCheckAuthsForMutation(false); for (HColumnDescriptor family : families) { tableDescriptor.addFamily(family); }