diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/Authorizations.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/Authorizations.java index 006bd6d..d7a25ba 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/Authorizations.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/Authorizations.java @@ -36,22 +36,11 @@ public class Authorizations { public Authorizations(String... labels) { this.labels = new ArrayList(labels.length); for (String label : labels) { - validateLabel(label); this.labels.add(label); } } - private void validateLabel(String label) { - if (!VisibilityLabelsValidator.isValidLabel(label)) { - throw new IllegalArgumentException("Invalid authorization label : " + label - + ". Authorizations cannot contain '(', ')' ,'&' ,'|', '!'" + " and cannot be empty"); - } - } - public Authorizations(List labels) { - for (String label : labels) { - validateLabel(label); - } this.labels = labels; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java index d91f0ef..b56f250 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java @@ -44,10 +44,10 @@ public final class VisibilityConstants { * Visibility serialization version format. It indicates the visibility labels * are sorted based on ordinal **/ - public static final byte VISIBILITY_SERIALIZATION_VERSION = 1; + public static final byte SORTED_ORDINAL_SERIALIZATION_FORMAT = 1; /** Byte representation of the visibility_serialization_version **/ - public static final byte[] SORTED_ORDINAL_SERIALIZATION_FORMAT = - new byte[] { VISIBILITY_SERIALIZATION_VERSION }; + public static final byte[] SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG_VAL = + new byte[] { SORTED_ORDINAL_SERIALIZATION_FORMAT }; public static final String CHECK_AUTHS_FOR_MUTATION = "hbase.security.visibility.mutations.checkauths"; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java index 97c5f0f..2528dd5 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java @@ -110,61 +110,5 @@ public class TestScan { Set qualifiers = scan.getFamilyMap().get(family); Assert.assertEquals(1, qualifiers.size()); } - - @Test - public void testSetAuthorizations() { - Scan scan = new Scan(); - scan.setAuthorizations(new Authorizations("A", "B", "0123", "A0", "1A1", "_a")); - try { - scan.setAuthorizations(new Authorizations("A|B")); - fail("Should have failed for A|B."); - } catch (IllegalArgumentException e) { - } - try { - scan.setAuthorizations(new Authorizations("A&B")); - fail("Should have failed for A&B."); - } catch (IllegalArgumentException e) { - } - try { - scan.setAuthorizations(new Authorizations("!B")); - fail("Should have failed for !B."); - } catch (IllegalArgumentException e) { - } - try { - scan.setAuthorizations(new Authorizations("A", "(A)")); - fail("Should have failed for (A)."); - } catch (IllegalArgumentException e) { - } - try { - scan.setAuthorizations(new Authorizations("A", "{A")); - fail("Should have failed for {A."); - } catch (IllegalArgumentException e) { - } - try { - scan.setAuthorizations(new Authorizations(" ")); - fail("Should have failed for empty"); - } catch (IllegalArgumentException e) { - } - try { - scan.setAuthorizations(new Authorizations(":B")); - } catch (IllegalArgumentException e) { - fail("Should not have failed for :B"); - } - try { - scan.setAuthorizations(new Authorizations("-B")); - } catch (IllegalArgumentException e) { - fail("Should not have failed for -B"); - } - try { - scan.setAuthorizations(new Authorizations(".B")); - } catch (IllegalArgumentException e) { - fail("Should not have failed for .B"); - } - try { - scan.setAuthorizations(new Authorizations("/B")); - } catch (IllegalArgumentException e) { - fail("Should not have failed for /B"); - } - } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java index 68018c4..a21f7de 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java @@ -27,5 +27,5 @@ public final class TagType { public static final byte ACL_TAG_TYPE = (byte) 1; public static final byte VISIBILITY_TAG_TYPE = (byte) 2; public static final byte LOG_REPLAY_TAG_TYPE = (byte) 3; - public static final byte VISIBILITY_EXP_SERIALIZATION_TAG_TYPE = (byte)4; + public static final byte VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE = (byte)4; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java index 60a3bc3..e4d4fe9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java @@ -33,6 +33,7 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.client.HTable; @@ -85,11 +86,11 @@ public class LabelExpander { // We will be adding this tag before the visibility tags and the presence of // this // tag indicates we are supporting deletes with cell visibility - tags.add(VisibilityUtils.VIS_SERIALIZATION_TAG); + tags.add(VisibilityUtils.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG); if (node.isSingleNode()) { getLabelOrdinals(node, labelOrdinals); writeLabelOrdinalsToStream(labelOrdinals, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + tags.add(new Tag(TagType.VISIBILITY_TAG_TYPE, baos.toByteArray())); baos.reset(); } else { NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node; @@ -97,14 +98,14 @@ public class LabelExpander { for (ExpressionNode child : nlNode.getChildExps()) { getLabelOrdinals(child, labelOrdinals); writeLabelOrdinalsToStream(labelOrdinals, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + tags.add(new Tag(TagType.VISIBILITY_TAG_TYPE, baos.toByteArray())); baos.reset(); labelOrdinals.clear(); } } else { getLabelOrdinals(nlNode, labelOrdinals); writeLabelOrdinalsToStream(labelOrdinals, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + tags.add(new Tag(TagType.VISIBILITY_TAG_TYPE, baos.toByteArray())); baos.reset(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java index e78f27a..d0639fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java @@ -47,16 +47,24 @@ public class OperationStatus { private final String exceptionMsg; + private final Exception exception; + public OperationStatus(OperationStatusCode code) { this(code, ""); } public OperationStatus(OperationStatusCode code, String exceptionMsg) { this.code = code; + this.exception = null; this.exceptionMsg = exceptionMsg; } - + public OperationStatus(OperationStatusCode code, Exception e) { + this.code = code; + this.exception = e; + this.exceptionMsg = e.getClass().getName() + ": " + e.getMessage(); + } + /** * @return OperationStatusCode */ @@ -71,4 +79,10 @@ public class OperationStatus { return exceptionMsg; } + /** + * @return exception + */ + public Exception getException(){ + return this.exception; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java index 2fa53f4..679a453 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java @@ -70,7 +70,6 @@ import org.apache.hadoop.hbase.filter.WhileMatchFilter; import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; import org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner; import org.apache.hadoop.hbase.security.visibility.Authorizations; -import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsValidator; import org.apache.hadoop.hbase.util.Base64; import org.apache.hadoop.hbase.util.Bytes; @@ -527,11 +526,6 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable { if (authorizations != null) { List labels = authorizations.getLabels(); for (String label : labels) { - if (!VisibilityLabelsValidator.isValidLabel(label)) { - throw new IllegalArgumentException("Invalid authorization label : " + label - + ". Authorizations cannot contain '(', ')' ,'&' ,'|', '!'" + " " + - "and cannot be empty"); - } model.addLabel(label); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultScanLabelGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultScanLabelGenerator.java index 5e2368b..19477b1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultScanLabelGenerator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultScanLabelGenerator.java @@ -38,10 +38,10 @@ public class DefaultScanLabelGenerator implements ScanLabelGenerator { private Configuration conf; - private VisibilityLabelsManager labelsManager; + private VisibilityLabelsCache labelsCache; public DefaultScanLabelGenerator() { - this.labelsManager = VisibilityLabelsManager.get(); + this.labelsCache = VisibilityLabelsCache.get(); } @Override @@ -59,7 +59,7 @@ public class DefaultScanLabelGenerator implements ScanLabelGenerator { if (authorizations != null) { List labels = authorizations.getLabels(); String userName = user.getShortName(); - List auths = this.labelsManager.getAuths(userName); + List auths = this.labelsCache.getAuths(userName); return dropLabelsNotInUserAuths(labels, auths, userName); } return null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java new file mode 100644 index 0000000..72af093 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java @@ -0,0 +1,693 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE; +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY; +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER; +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT; +import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.io.util.StreamUtils; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.OperationStatus; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.security.AccessDeniedException; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode; +import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode; +import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode; +import org.apache.hadoop.hbase.security.visibility.expression.Operator; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; + +import com.google.common.collect.Lists; + +@InterfaceAudience.Private +public class DefaultVisibilityLabelServiceImpl implements VisibilityLabelService { + + private static final Log LOG = LogFactory.getLog(DefaultVisibilityLabelServiceImpl.class); + + // "system" label is having an ordinal value 1. + private static final int SYSTEM_LABEL_ORDINAL = 1; + private static final Tag[] LABELS_TABLE_TAGS = new Tag[1]; + private static final byte[] DUMMY_VALUE = new byte[0]; + + private volatile int ordinalCounter = -1; + private final ExpressionParser expressionParser = new ExpressionParser(); + private final ExpressionExpander expressionExpander = new ExpressionExpander(); + private Configuration conf; + private HRegion labelsRegion; + private VisibilityLabelsCache labelsCache; + private List scanLabelGenerators; + + static { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + try { + StreamUtils.writeRawVInt32(dos, SYSTEM_LABEL_ORDINAL); + } catch (IOException e) { + // We write to a byte array. No Exception can happen. + } + LABELS_TABLE_TAGS[0] = new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()); + } + + public DefaultVisibilityLabelServiceImpl() { + + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return this.conf; + } + + @Override + public void init(RegionCoprocessorEnvironment e) { + this.scanLabelGenerators = VisibilityUtils.getScanLabelGenerators(this.conf); + ZooKeeperWatcher zk = e.getRegionServerServices().getZooKeeper(); + try { + labelsCache = VisibilityLabelsCache.get(zk, this.conf); + } catch (IOException ioe) { + LOG.error("Error creating VisibilityLabelsManager", ioe); + throw new RuntimeException(ioe); + } + if (e.getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) { + this.labelsRegion = e.getRegion(); + try { + Pair, Map>> labelsAndUserAuths = + extractLabelsAndAuths(getExistingLabelsWithAuths()); + Map labels = labelsAndUserAuths.getFirst(); + Map> userAuths = labelsAndUserAuths.getSecond(); + // Add the "system" label if it is not added into the system yet + addSystemLabel(this.labelsRegion, labels, userAuths); + int ordinal = 1; // Ordinal 1 is reserved for "system" label. + for (Integer i : labels.values()) { + if (i > ordinal) { + ordinal = i; + } + } + this.ordinalCounter = ordinal + 1; + if (labels.size() > 0) { + // If there is no data need not write to zk + byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(labels); + this.labelsCache.writeToZookeeper(serialized, true); + this.labelsCache.refreshLabelsCache(serialized); + } + if (userAuths.size() > 0) { + byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths); + this.labelsCache.writeToZookeeper(serialized, false); + this.labelsCache.refreshUserAuthsCache(serialized); + } + } catch (IOException ioe) { + LOG.error("Error while updating the zk with the exisiting labels data", ioe); + } + } + } + + protected List> getExistingLabelsWithAuths() throws IOException { + Scan scan = new Scan(); + RegionScanner scanner = labelsRegion.getScanner(scan); + List> existingLabels = new ArrayList>(); + try { + while (true) { + List cells = new ArrayList(); + scanner.next(cells); + if (cells.isEmpty()) { + break; + } + existingLabels.add(cells); + } + } finally { + scanner.close(); + } + return existingLabels; + } + + protected Pair, Map>> extractLabelsAndAuths( + List> labelDetails) { + Map labels = new HashMap(); + Map> userAuths = new HashMap>(); + for (List cells : labelDetails) { + for (Cell cell : cells) { + if (Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(), + cell.getQualifierLength(), LABEL_QUALIFIER, 0, LABEL_QUALIFIER.length)) { + labels.put( + Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()), + Bytes.toInt(cell.getRowArray(), cell.getRowOffset())); + } else { + // These are user cells who has authorization for this label + String user = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), + cell.getQualifierLength()); + List auths = userAuths.get(user); + if (auths == null) { + auths = new ArrayList(); + userAuths.put(user, auths); + } + auths.add(Bytes.toInt(cell.getRowArray(), cell.getRowOffset())); + } + } + } + return new Pair, Map>>(labels, userAuths); + } + + protected void addSystemLabel(HRegion region, Map labels, + Map> userAuths) throws IOException { + if (!labels.containsKey(SYSTEM_LABEL)) { + Put p = new Put(Bytes.toBytes(SYSTEM_LABEL_ORDINAL)); + p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, Bytes.toBytes(SYSTEM_LABEL)); + // Set auth for "system" label for all super users. + List superUsers = getSystemAndSuperUsers(); + for (String superUser : superUsers) { + p.addImmutable(LABELS_TABLE_FAMILY, Bytes.toBytes(superUser), DUMMY_VALUE, + LABELS_TABLE_TAGS); + } + region.put(p); + labels.put(SYSTEM_LABEL, SYSTEM_LABEL_ORDINAL); + for (String superUser : superUsers) { + List auths = userAuths.get(superUser); + if (auths == null) { + auths = new ArrayList(1); + userAuths.put(superUser, auths); + } + auths.add(SYSTEM_LABEL_ORDINAL); + } + } + } + + protected List getSystemAndSuperUsers() throws IOException { + User user = User.getCurrent(); + if (user == null) { + throw new IOException("Unable to obtain the current user, " + + "authorization checks for internal operations will not work correctly!"); + } + if (LOG.isTraceEnabled()) { + LOG.trace("Current user name is " + user.getShortName()); + } + String currentUser = user.getShortName(); + List superUsers = Lists.asList(currentUser, + this.conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0])); + return superUsers; + } + + @Override + public OperationStatus[] addLabels(List labels) throws IOException { + assert labelsRegion != null; + OperationStatus[] finalOpStatus = new OperationStatus[labels.size()]; + List puts = new ArrayList(labels.size()); + int i = 0; + for (byte[] label : labels) { + String labelStr = Bytes.toString(label); + if (VisibilityLabelsValidator.isValidLabel(label)) { + if (this.labelsCache.getLabelOrdinal(labelStr) > 0) { + finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE, + new LabelAlreadyExistsException("Label '" + labelStr + "' already exists")); + } else { + Put p = new Put(Bytes.toBytes(ordinalCounter)); + p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, label, LABELS_TABLE_TAGS); + if (LOG.isDebugEnabled()) { + LOG.debug("Adding the label " + labelStr); + } + puts.add(p); + ordinalCounter++; + } + } else { + finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE, + new InvalidLabelException("Invalid visibility label '" + labelStr + "'")); + } + i++; + } + if (mutateLabelsRegion(puts, finalOpStatus)) { + updateZk(true); + } + return finalOpStatus; + } + + @Override + public OperationStatus[] setAuths(byte[] user, List authLabels) throws IOException { + assert labelsRegion != null; + OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()]; + List puts = new ArrayList(authLabels.size()); + int i = 0; + for (byte[] auth : authLabels) { + String authStr = Bytes.toString(auth); + int labelOrdinal = this.labelsCache.getLabelOrdinal(authStr); + if (labelOrdinal == 0) { + // This label is not yet added. 1st this should be added to the system + finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE, + new InvalidLabelException("Label '" + authStr + "' doesn't exists")); + } else { + Put p = new Put(Bytes.toBytes(labelOrdinal)); + p.addImmutable(LABELS_TABLE_FAMILY, user, DUMMY_VALUE, LABELS_TABLE_TAGS); + puts.add(p); + } + i++; + } + if (mutateLabelsRegion(puts, finalOpStatus)) { + updateZk(false); + } + return finalOpStatus; + } + + @Override + public OperationStatus[] clearAuths(byte[] user, List authLabels) throws IOException { + assert labelsRegion != null; + OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()]; + List currentAuths = this.getAuths(user, true); + List deletes = new ArrayList(authLabels.size()); + int i = 0; + for (byte[] authLabel : authLabels) { + String authLabelStr = Bytes.toString(authLabel); + if (currentAuths.contains(authLabelStr)) { + int labelOrdinal = this.labelsCache.getLabelOrdinal(authLabelStr); + assert labelOrdinal > 0; + Delete d = new Delete(Bytes.toBytes(labelOrdinal)); + d.deleteColumns(LABELS_TABLE_FAMILY, user); + deletes.add(d); + } else { + // This label is not set for the user. + finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE, + new InvalidLabelException("Label '" + authLabelStr + "' is not set for the user " + + Bytes.toString(user))); + } + i++; + } + if (mutateLabelsRegion(deletes, finalOpStatus)) { + updateZk(false); + } + return finalOpStatus; + } + + /* + * Adds the mutations to labels region and set the results to the finalOpStatus. finalOpStatus + * might have some entries in it where the OpStatus is FAILURE. We will leave those and set in + * others in the order. + * @param mutations + * @param finalOpStatus + * @return whether we need a ZK update or not. + */ + private boolean mutateLabelsRegion(List mutations, OperationStatus[] finalOpStatus) + throws IOException { + OperationStatus[] opStatus = this.labelsRegion.batchMutate(mutations + .toArray(new Mutation[mutations.size()])); + int i = 0; + boolean updateZk = false; + for (OperationStatus status : opStatus) { + // Update the zk when atleast one of the mutation was added successfully. + updateZk = updateZk || (status.getOperationStatusCode() == OperationStatusCode.SUCCESS); + for (; i < finalOpStatus.length; i++) { + if (finalOpStatus[i] == null) { + finalOpStatus[i] = status; + break; + } + } + } + return updateZk; + } + + @Override + public List getAuths(byte[] user, boolean systemCall) throws IOException { + assert (labelsRegion != null || systemCall); + if (systemCall || labelsRegion == null) { + return this.labelsCache.getAuths(Bytes.toString(user)); + } + Scan s = new Scan(); + s.addColumn(LABELS_TABLE_FAMILY, user); + Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion, + new Authorizations(SYSTEM_LABEL), this); + s.setFilter(filter); + List auths = new ArrayList(); + RegionScanner scanner = this.labelsRegion.getScanner(s); + List results = new ArrayList(1); + while (true) { + scanner.next(results); + if (results.isEmpty()) break; + Cell cell = results.get(0); + int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); + String label = this.labelsCache.getLabel(ordinal); + if (label != null) { + auths.add(label); + } + results.clear(); + } + return auths; + } + + @Override + public List createVisibilityExpTags(String visExpression, boolean withSerializationFormat, + boolean checkAuths) throws IOException { + ExpressionNode node = null; + try { + node = this.expressionParser.parse(visExpression); + } catch (ParseException e) { + throw new IOException(e); + } + node = this.expressionExpander.expand(node); + List tags = new ArrayList(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + List labelOrdinals = new ArrayList(); + // We will be adding this tag before the visibility tags and the presence of this + // tag indicates we are supporting deletes with cell visibility + if (withSerializationFormat) { + tags.add(VisibilityUtils.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG); + } + Set auths = null; + if (checkAuths) { + auths = this.labelsCache.getAuthsAsOrdinals(VisibilityUtils.getActiveUser().getShortName()); + } + if (node.isSingleNode()) { + getLabelOrdinals(node, labelOrdinals, auths, checkAuths); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray())); + baos.reset(); + } else { + NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node; + if (nlNode.getOperator() == Operator.OR) { + for (ExpressionNode child : nlNode.getChildExps()) { + getLabelOrdinals(child, labelOrdinals, auths, checkAuths); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray())); + baos.reset(); + labelOrdinals.clear(); + } + } else { + getLabelOrdinals(nlNode, labelOrdinals, auths, checkAuths); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray())); + baos.reset(); + } + } + return tags; + } + + protected void getLabelOrdinals(ExpressionNode node, List labelOrdinals, + Set auths, boolean checkAuths) throws IOException, InvalidLabelException { + if (node.isSingleNode()) { + String identifier = null; + int labelOrdinal = 0; + if (node instanceof LeafExpressionNode) { + identifier = ((LeafExpressionNode) node).getIdentifier(); + if (LOG.isTraceEnabled()) { + LOG.trace("The identifier is " + identifier); + } + labelOrdinal = this.labelsCache.getLabelOrdinal(identifier); + checkAuths(auths, labelOrdinal, identifier, checkAuths); + } else { + // This is a NOT node. + LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node) + .getChildExps().get(0); + identifier = lNode.getIdentifier(); + labelOrdinal = this.labelsCache.getLabelOrdinal(identifier); + checkAuths(auths, labelOrdinal, identifier, checkAuths); + labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal. + } + if (labelOrdinal == 0) { + throw new InvalidLabelException("Invalid visibility label " + identifier); + } + labelOrdinals.add(labelOrdinal); + } else { + List childExps = ((NonLeafExpressionNode) node).getChildExps(); + for (ExpressionNode child : childExps) { + getLabelOrdinals(child, labelOrdinals, auths, checkAuths); + } + } + } + + private void checkAuths(Set auths, int labelOrdinal, String identifier, + boolean checkAuths) throws IOException { + if (checkAuths) { + if (auths == null || (!auths.contains(labelOrdinal))) { + throw new AccessDeniedException("Visibility label " + identifier + + " not authorized for the user " + VisibilityUtils.getActiveUser().getShortName()); + } + } + } + + protected void writeLabelOrdinalsToStream(List labelOrdinals, DataOutputStream dos) + throws IOException { + Collections.sort(labelOrdinals); + for (Integer labelOrdinal : labelOrdinals) { + StreamUtils.writeRawVInt32(dos, labelOrdinal); + } + } + + protected void updateZk(boolean labelAddition) throws IOException { + // We will add to zookeeper here. + // TODO we should add the delta only to zk. Else this will be a very heavy op and when there are + // so many labels and auth in the system, we will end up adding lots of data to zk. Most + // possibly we will exceed zk node data limit! + // Will correct this soon. + Pair, Map>> labelsAndUserAuths = + extractLabelsAndAuths(getExistingLabelsWithAuths()); + Map existingLabels = labelsAndUserAuths.getFirst(); + Map> userAuths = labelsAndUserAuths.getSecond(); + if (labelAddition) { + byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(existingLabels); + this.labelsCache.writeToZookeeper(serialized, true); + } else { + byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths); + this.labelsCache.writeToZookeeper(serialized, false); + } + } + + @Override + public VisibilityExpEvaluator getVisibilityExpEvaluator(Authorizations authorizations) + throws IOException { + // If a super user issues a get/scan, he should be able to scan the cells + // irrespective of the Visibility labels + if (isReadFromSuperUser()) { + return new VisibilityExpEvaluator() { + @Override + public boolean evaluate(Cell cell) throws IOException { + return true; + } + }; + } + if (authorizations != null) { + for (String label : authorizations.getLabels()) { + if (!VisibilityLabelsValidator.isValidLabel(label)) { + throw new DoNotRetryIOException("Invalid authorization label : " + label + + ". Authorizations cannot contain '(', ')' ,'&' ,'|', '!'" + " and cannot be empty"); + } + } + } + List authLabels = null; + for (ScanLabelGenerator scanLabelGenerator : scanLabelGenerators) { + try { + // null authorizations to be handled inside SLG impl. + authLabels = scanLabelGenerator.getLabels(VisibilityUtils.getActiveUser(), authorizations); + authLabels = (authLabels == null) ? new ArrayList() : authLabels; + authorizations = new Authorizations(authLabels); + } catch (Throwable t) { + LOG.error(t); + throw new IOException(t); + } + } + int labelsCount = this.labelsCache.getLabelsCount(); + final BitSet bs = new BitSet(labelsCount + 1); // ordinal is index 1 based + if (authLabels != null) { + for (String authLabel : authLabels) { + int labelOrdinal = this.labelsCache.getLabelOrdinal(authLabel); + if (labelOrdinal != 0) { + bs.set(labelOrdinal); + } + } + } + + return new VisibilityExpEvaluator() { + @Override + public boolean evaluate(Cell cell) throws IOException { + Iterator tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), + cell.getTagsLength()); + boolean visibilityTagPresent = false; + while (tagsItr.hasNext()) { + boolean includeKV = true; + Tag tag = tagsItr.next(); + if (tag.getType() == VISIBILITY_TAG_TYPE) { + visibilityTagPresent = true; + int offset = tag.getTagOffset(); + int endOffset = offset + tag.getTagLength(); + while (offset < endOffset) { + Pair result = StreamUtils.readRawVarint32(tag.getBuffer(), offset); + int currLabelOrdinal = result.getFirst(); + if (currLabelOrdinal < 0) { + // check for the absence of this label in the Scan Auth labels + // ie. to check BitSet corresponding bit is 0 + int temp = -currLabelOrdinal; + if (bs.get(temp)) { + includeKV = false; + break; + } + } else { + if (!bs.get(currLabelOrdinal)) { + includeKV = false; + break; + } + } + offset += result.getSecond(); + } + if (includeKV) { + // We got one visibility expression getting evaluated to true. Good to include this KV + // in the result then. + return true; + } + } + } + return !(visibilityTagPresent); + } + }; + } + + protected boolean isReadFromSuperUser() throws IOException { + byte[] user = Bytes.toBytes(VisibilityUtils.getActiveUser().getShortName()); + return havingSystemAuth(user); + } + + @Override + public boolean havingSystemAuth(byte[] user) throws IOException { + List auths = this.getAuths(user, true); + if (LOG.isTraceEnabled()) { + LOG.trace("The auths for user " + Bytes.toString(user) + " are " + auths); + } + return auths.contains(SYSTEM_LABEL); + } + + @Override + public boolean matchVisibility(List putVisTags, Byte putTagsFormat, List deleteVisTags, + Byte deleteTagsFormat) throws IOException { + if ((deleteTagsFormat != null && deleteTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT) + && (putTagsFormat == null || putTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)) { + if (putVisTags.size() == 0) { + // Early out if there are no tags in the cell + return false; + } + if (putTagsFormat == null) { + return checkForMatchingVisibilityTagsWithOutSortedOrder(putVisTags, deleteVisTags); + } else { + return checkForMatchingVisibilityTagsWithSortedOrder(putVisTags, deleteVisTags); + } + } + throw new IOException("Unexpected tag format passed for comparison, deleteTagsFormat : " + + deleteTagsFormat + ", putTagsFormat : " + putTagsFormat); + } + + private static boolean checkForMatchingVisibilityTagsWithOutSortedOrder(List putVisTags, + List deleteVisTags) throws IOException { + return compareTagsOrdinals(sortTagsBasedOnOrdinal(putVisTags), + sortTagsBasedOnOrdinal(deleteVisTags)); + } + + private static boolean checkForMatchingVisibilityTagsWithSortedOrder(List putVisTags, + List deleteVisTags) { + boolean matchFound = false; + if ((deleteVisTags.size()) != putVisTags.size()) { + // If the size does not match. Definitely we are not comparing the equal tags. + // Return false in that case. + return matchFound; + } + for (Tag tag : deleteVisTags) { + matchFound = false; + for (Tag givenTag : putVisTags) { + if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(), + givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) { + matchFound = true; + break; + } + } + } + return matchFound; + } + + private static List> sortTagsBasedOnOrdinal(List tags) throws IOException { + List> fullTagsList = new ArrayList>(); + for (Tag tag : tags) { + if (tag.getType() == VISIBILITY_TAG_TYPE) { + getSortedTagOrdinals(fullTagsList, tag); + } + } + return fullTagsList; + } + + private static void getSortedTagOrdinals(List> fullTagsList, Tag tag) + throws IOException { + List tagsOrdinalInSortedOrder = new ArrayList(); + int offset = tag.getTagOffset(); + int endOffset = offset + tag.getTagLength(); + while (offset < endOffset) { + Pair result = StreamUtils.readRawVarint32(tag.getBuffer(), offset); + tagsOrdinalInSortedOrder.add(result.getFirst()); + offset += result.getSecond(); + } + Collections.sort(tagsOrdinalInSortedOrder); + fullTagsList.add(tagsOrdinalInSortedOrder); + } + + private static boolean compareTagsOrdinals(List> putVisTags, + List> deleteVisTags) { + boolean matchFound = false; + if (deleteVisTags.size() != putVisTags.size()) { + return matchFound; + } else { + for (List deleteTagOrdinals : deleteVisTags) { + matchFound = false; + for (List tagOrdinals : putVisTags) { + if (deleteTagOrdinals.equals(tagOrdinals)) { + matchFound = true; + break; + } + } + } + return matchFound; + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/EnforcingScanLabelGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/EnforcingScanLabelGenerator.java index 7d0320a..90a8ff9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/EnforcingScanLabelGenerator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/EnforcingScanLabelGenerator.java @@ -37,10 +37,10 @@ public class EnforcingScanLabelGenerator implements ScanLabelGenerator { private static final Log LOG = LogFactory.getLog(EnforcingScanLabelGenerator.class); private Configuration conf; - private VisibilityLabelsManager labelsManager; + private VisibilityLabelsCache labelsCache; public EnforcingScanLabelGenerator() { - this.labelsManager = VisibilityLabelsManager.get(); + this.labelsCache = VisibilityLabelsCache.get(); } @Override @@ -59,7 +59,7 @@ public class EnforcingScanLabelGenerator implements ScanLabelGenerator { if (authorizations != null) { LOG.warn("Dropping authorizations requested by user " + userName + ": " + authorizations); } - return this.labelsManager.getAuths(userName); + return this.labelsCache.getAuths(userName); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index 7a4481e..76308cf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -22,20 +22,13 @@ import static org.apache.hadoop.hbase.HConstants.OperationStatusCode.SANITY_CHEC import static org.apache.hadoop.hbase.HConstants.OperationStatusCode.SUCCESS; import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY; import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; -import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER; -import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL; -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -58,6 +51,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -83,7 +77,6 @@ import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterBase; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.ipc.RequestContext; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; @@ -110,16 +103,8 @@ import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessControlLists; import org.apache.hadoop.hbase.security.access.AccessController; -import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode; -import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode; -import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode; -import org.apache.hadoop.hbase.security.visibility.expression.Operator; -import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.util.SimpleMutableByteRange; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import com.google.common.collect.Lists; import com.google.common.collect.MapMaker; @@ -137,19 +122,6 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb RegionObserver, VisibilityLabelsService.Interface, CoprocessorService { private static final Log LOG = LogFactory.getLog(VisibilityController.class); - private static final byte[] DUMMY_VALUE = new byte[0]; - // "system" label is having an ordinal value 1. - private static final int SYSTEM_LABEL_ORDINAL = 1; - private static final Tag[] LABELS_TABLE_TAGS = new Tag[1]; - - private final ExpressionParser expressionParser = new ExpressionParser(); - private final ExpressionExpander expressionExpander = new ExpressionExpander(); - private VisibilityLabelsManager visibilityManager; - // defined only for Endpoint implementation, so it can have way to access region services. - private RegionCoprocessorEnvironment regionEnv; - private List scanLabelGenerators; - - private volatile int ordinalCounter = -1; // flags if we are running on a region of the 'labels' table private boolean labelsRegion = false; // Flag denoting whether AcessController is available or not. @@ -162,23 +134,13 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb new MapMaker().weakKeys().makeMap(); List superUsers; - - static { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - try { - StreamUtils.writeRawVInt32(dos, SYSTEM_LABEL_ORDINAL); - } catch (IOException e) { - // We write to a byte array. No Exception can happen. - } - LABELS_TABLE_TAGS[0] = new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()); - } + private VisibilityLabelService visibilityLabelService; // Add to this list if there are any reserved tag types - private static ArrayList reservedVisTagTypes = new ArrayList(); + private static ArrayList RESERVED_VIS_TAG_TYPES = new ArrayList(); static { - reservedVisTagTypes.add(VisibilityUtils.VISIBILITY_TAG_TYPE); - reservedVisTagTypes.add(VisibilityUtils.VISIBILITY_EXP_SERIALIZATION_TAG_TYPE); + RESERVED_VIS_TAG_TYPES.add(TagType.VISIBILITY_TAG_TYPE); + RESERVED_VIS_TAG_TYPES.add(TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE); } @Override @@ -189,34 +151,15 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb + " is required to persist visibility labels. Consider setting " + HFile.FORMAT_VERSION_KEY + " accordingly."); } - ZooKeeperWatcher zk = null; - if (env instanceof MasterCoprocessorEnvironment) { - // if running on HMaster - MasterCoprocessorEnvironment mEnv = (MasterCoprocessorEnvironment) env; - zk = mEnv.getMasterServices().getZooKeeper(); - } else if (env instanceof RegionCoprocessorEnvironment) { - // if running at region - regionEnv = (RegionCoprocessorEnvironment) env; - zk = regionEnv.getRegionServerServices().getZooKeeper(); - } else if (env instanceof RegionServerCoprocessorEnvironment) { + if (env instanceof RegionServerCoprocessorEnvironment) { throw new RuntimeException( "Visibility controller should not be configured as " + "'hbase.coprocessor.regionserver.classes'."); } - - // If zk is null or IOException while obtaining auth manager, - // throw RuntimeException so that the coprocessor is unloaded. - if (zk == null) { - throw new RuntimeException("Error obtaining VisibilityLabelsManager, zk found null."); - } - try { - this.visibilityManager = VisibilityLabelsManager.get(zk, this.conf); - } catch (IOException ioe) { - throw new RuntimeException("Error obtaining VisibilityLabelsManager", ioe); - } if (env instanceof RegionCoprocessorEnvironment) { - // ScanLabelGenerator to be instantiated only with Region Observer. - scanLabelGenerators = VisibilityUtils.getScanLabelGenerators(this.conf); + // VisibilityLabelService to be instantiated only with Region Observer. + visibilityLabelService = VisibilityLabelServiceManager.getInstance() + .getVisibilityLabelService(this.conf); } this.superUsers = getSystemAndSuperUsers(); } @@ -618,12 +561,15 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (e.getEnvironment().getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) { this.labelsRegion = true; this.acOn = CoprocessorHost.getLoadedCoprocessors().contains(AccessController.class.getName()); + // Defer the init of VisibilityLabelService on labels region until it is in recovering state. if (!e.getEnvironment().getRegion().isRecovering()) { - initialize(e); + this.visibilityLabelService.init(e.getEnvironment()); + this.initialized = true; } } else { checkAuths = e.getEnvironment().getConfiguration() .getBoolean(VisibilityConstants.CHECK_AUTHS_FOR_MUTATION, false); + this.visibilityLabelService.init(e.getEnvironment()); this.initialized = true; } } @@ -631,60 +577,8 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb @Override public void postLogReplay(ObserverContext e) { if (this.labelsRegion) { - initialize(e); - } - } - - private void initialize(ObserverContext e) { - try { - Pair, Map>> labelsAndUserAuths = - extractLabelsAndAuths(getExistingLabelsWithAuths()); - Map labels = labelsAndUserAuths.getFirst(); - Map> userAuths = labelsAndUserAuths.getSecond(); - // Add the "system" label if it is not added into the system yet - addSystemLabel(e.getEnvironment().getRegion(), labels, userAuths); - int ordinal = 1; // Ordinal 1 is reserved for "system" label. - for (Integer i : labels.values()) { - if (i > ordinal) { - ordinal = i; - } - } - this.ordinalCounter = ordinal + 1; - if (labels.size() > 0) { - // If there is no data need not write to zk - byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(labels); - this.visibilityManager.writeToZookeeper(serialized, true); - } - if (userAuths.size() > 0) { - byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths); - this.visibilityManager.writeToZookeeper(serialized, false); - } - initialized = true; - } catch (IOException ioe) { - LOG.error("Error while updating the zk with the exisiting labels data", ioe); - } - } - - private void addSystemLabel(HRegion region, Map labels, - Map> userAuths) throws IOException { - if (!labels.containsKey(SYSTEM_LABEL)) { - Put p = new Put(Bytes.toBytes(SYSTEM_LABEL_ORDINAL)); - p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, Bytes.toBytes(SYSTEM_LABEL)); - // Set auth for "system" label for all super users. - for (String superUser : this.superUsers) { - p.addImmutable( - LABELS_TABLE_FAMILY, Bytes.toBytes(superUser), DUMMY_VALUE, LABELS_TABLE_TAGS); - } - region.put(p); - labels.put(SYSTEM_LABEL, SYSTEM_LABEL_ORDINAL); - for (String superUser : superUsers) { - List auths = userAuths.get(superUser); - if (auths == null) { - auths = new ArrayList(1); - userAuths.put(superUser, auths); - } - auths.add(SYSTEM_LABEL_ORDINAL); - } + this.visibilityLabelService.init(e.getEnvironment()); + this.initialized = true; } } @@ -696,11 +590,6 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } // TODO this can be made as a global LRU cache at HRS level? Map> labelCache = new HashMap>(); - Set auths = null; - User user = getActiveUser(); - if (checkAuths && user != null && user.getShortName() != null) { - auths = this.visibilityManager.getAuthsAsOrdinals(user.getShortName()); - } for (int i = 0; i < miniBatchOp.size(); i++) { Mutation m = miniBatchOp.getOperation(i); CellVisibility cellVisibility = null; @@ -711,46 +600,49 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb new OperationStatus(SANITY_CHECK_FAILURE, de.getMessage())); continue; } - boolean sanityFailure = false; - for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { - if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { - miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, - "Mutation contains cell with reserved type tag")); - sanityFailure = true; - break; - } + boolean sanityFailure = false; + for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { + if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { + miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, + "Mutation contains cell with reserved type tag")); + sanityFailure = true; + break; } - if (!sanityFailure) { - if (cellVisibility != null) { - String labelsExp = cellVisibility.getExpression(); - List visibilityTags = labelCache.get(labelsExp); - if (visibilityTags == null) { - try { - visibilityTags = createVisibilityTags(labelsExp, true, auths, user.getShortName()); - } catch (ParseException e) { - miniBatchOp.setOperationStatus(i, - new OperationStatus(SANITY_CHECK_FAILURE, e.getMessage())); - } catch (InvalidLabelException e) { - miniBatchOp.setOperationStatus(i, - new OperationStatus(SANITY_CHECK_FAILURE, e.getMessage())); + } + if (!sanityFailure) { + if (cellVisibility != null) { + String labelsExp = cellVisibility.getExpression(); + List visibilityTags = labelCache.get(labelsExp); + if (visibilityTags == null) { + try { + // Don't check user auths for labels with Mutations when the user is super user + boolean authCheck = this.checkAuths && !(isSystemOrSuperUser()); + visibilityTags = this.visibilityLabelService.createVisibilityExpTags(labelsExp, true, + authCheck); + if (visibilityTags != null) { + labelCache.put(labelsExp, visibilityTags); } + } catch (InvalidLabelException e) { + miniBatchOp.setOperationStatus(i, + new OperationStatus(SANITY_CHECK_FAILURE, e.getMessage())); + } + } + if (visibilityTags != null) { + labelCache.put(labelsExp, visibilityTags); + List updatedCells = new ArrayList(); + for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { + Cell cell = cellScanner.current(); + List tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), + cell.getTagsLength()); + tags.addAll(visibilityTags); + Cell updatedCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(), + cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(), + cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), + cell.getQualifierLength(), cell.getTimestamp(), Type.codeToType(cell + .getTypeByte()), cell.getValueArray(), cell.getValueOffset(), + cell.getValueLength(), tags); + updatedCells.add(updatedCell); } - if (visibilityTags != null) { - labelCache.put(labelsExp, visibilityTags); - List updatedCells = new ArrayList(); - for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { - Cell cell = cellScanner.current(); - List tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), - cell.getTagsLength()); - tags.addAll(visibilityTags); - Cell updatedCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(), - cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(), - cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength(), cell.getTimestamp(), Type.codeToType(cell - .getTypeByte()), cell.getValueArray(), cell.getValueOffset(), - cell.getValueLength(), tags); - updatedCells.add(updatedCell); - } m.getFamilyCellMap().clear(); // Clear and add new Cells to the Mutation. for (Cell cell : updatedCells) { @@ -786,14 +678,14 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (cellVisibility != null) { String labelsExp = cellVisibility.getExpression(); try { - visibilityTags = createVisibilityTags(labelsExp, false, null, null); - } catch (ParseException e) { - throw new IOException("Invalid cell visibility expression " + labelsExp, e); + visibilityTags = this.visibilityLabelService.createVisibilityExpTags(labelsExp, false, + false); } catch (InvalidLabelException e) { throw new IOException("Invalid cell visibility specified " + labelsExp, e); } } - get.setFilter(new DeleteVersionVisibilityExpressionFilter(visibilityTags)); + get.setFilter(new DeleteVersionVisibilityExpressionFilter(visibilityTags, + VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT)); List result = ctx.getEnvironment().getRegion().get(get, false); if (result.size() < get.getMaxVersions()) { @@ -816,89 +708,6 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb ctx.bypass(); } - @Override - public void postBatchMutate(ObserverContext c, - MiniBatchOperationInProgress miniBatchOp) throws IOException { - if (this.labelsRegion) { - // We will add to zookeeper here. - Pair, Map>> labelsAndUserAuths = - extractLabelsAndAuths(getExistingLabelsWithAuths()); - Map existingLabels = labelsAndUserAuths.getFirst(); - Map> userAuths = labelsAndUserAuths.getSecond(); - boolean isNewLabels = false; - boolean isUserAuthsChange = false; - for (int i = 0; i < miniBatchOp.size(); i++) { - Mutation m = miniBatchOp.getOperation(i); - if (miniBatchOp.getOperationStatus(i).getOperationStatusCode() == SUCCESS) { - for (List cells : m.getFamilyCellMap().values()) { - for (Cell cell : cells) { - int labelOrdinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset()); - if (Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength(), LABEL_QUALIFIER, 0, - LABEL_QUALIFIER.length)) { - if (m instanceof Put) { - existingLabels.put( - Bytes.toString(cell.getValueArray(), cell.getValueOffset(), - cell.getValueLength()), labelOrdinal); - isNewLabels = true; - } - } else { - String user = Bytes.toString(cell.getQualifierArray(), - cell.getQualifierOffset(), cell.getQualifierLength()); - List auths = userAuths.get(user); - if (auths == null) { - auths = new ArrayList(); - userAuths.put(user, auths); - } - if (m instanceof Delete) { - auths.remove(Integer.valueOf(labelOrdinal)); - } else { - auths.add(labelOrdinal); - } - isUserAuthsChange = true; - } - } - } - } - } - if (isNewLabels) { - byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(existingLabels); - this.visibilityManager.writeToZookeeper(serialized, true); - } - if (isUserAuthsChange) { - byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths); - this.visibilityManager.writeToZookeeper(serialized, false); - } - } - } - - private Pair, Map>> extractLabelsAndAuths( - List> labelDetails) { - Map labels = new HashMap(); - Map> userAuths = new HashMap>(); - for (List cells : labelDetails) { - for (Cell cell : cells) { - if (Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength(), LABEL_QUALIFIER, 0, LABEL_QUALIFIER.length)) { - labels.put( - Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()), - Bytes.toInt(cell.getRowArray(), cell.getRowOffset())); - } else { - // These are user cells who has authorization for this label - String user = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength()); - List auths = userAuths.get(user); - if (auths == null) { - auths = new ArrayList(); - userAuths.put(user, auths); - } - auths.add(Bytes.toInt(cell.getRowArray(), cell.getRowOffset())); - } - } - } - return new Pair, Map>>(labels, userAuths); - } - // Checks whether cell contains any tag with type as VISIBILITY_TAG_TYPE. // This tag type is reserved and should not be explicitly set by user. private boolean checkForReservedVisibilityTagPresence(Cell cell) throws IOException { @@ -912,7 +721,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb Iterator tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsItr.hasNext()) { - if (reservedVisTagTypes.contains(tagsItr.next().getType())) { + if (RESERVED_VIS_TAG_TYPES.contains(tagsItr.next().getType())) { return false; } } @@ -920,114 +729,29 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb return true; } - private List createVisibilityTags(String visibilityLabelsExp, boolean addSerializationTag, - Set auths, String userName) throws IOException, ParseException, - InvalidLabelException { - ExpressionNode node = null; - node = this.expressionParser.parse(visibilityLabelsExp); - node = this.expressionExpander.expand(node); - List tags = new ArrayList(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - List labelOrdinals = new ArrayList(); - // We will be adding this tag before the visibility tags and the presence of this - // tag indicates we are supporting deletes with cell visibility - if (addSerializationTag) { - tags.add(VisibilityUtils.VIS_SERIALIZATION_TAG); - } - if (node.isSingleNode()) { - getLabelOrdinals(node, labelOrdinals, auths, userName); - writeLabelOrdinalsToStream(labelOrdinals, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); - baos.reset(); - } else { - NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node; - if (nlNode.getOperator() == Operator.OR) { - for (ExpressionNode child : nlNode.getChildExps()) { - getLabelOrdinals(child, labelOrdinals, auths, userName); - writeLabelOrdinalsToStream(labelOrdinals, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); - baos.reset(); - labelOrdinals.clear(); - } - } else { - getLabelOrdinals(nlNode, labelOrdinals, auths, userName); - writeLabelOrdinalsToStream(labelOrdinals, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); - baos.reset(); - } - } - return tags; - } - - private void writeLabelOrdinalsToStream(List labelOrdinals, DataOutputStream dos) - throws IOException { - Collections.sort(labelOrdinals); - for (Integer labelOrdinal : labelOrdinals) { - StreamUtils.writeRawVInt32(dos, labelOrdinal); - } - } - - private void getLabelOrdinals(ExpressionNode node, List labelOrdinals, - Set auths, String userName) throws IOException, InvalidLabelException { - if (node.isSingleNode()) { - String identifier = null; - int labelOrdinal = 0; - if (node instanceof LeafExpressionNode) { - identifier = ((LeafExpressionNode) node) - .getIdentifier(); - if (LOG.isTraceEnabled()) { - LOG.trace("The identifier is "+identifier); - } - labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier); - checkAuths(auths, userName, labelOrdinal, identifier); - } else { - // This is a NOT node. - LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node) - .getChildExps().get(0); - identifier = lNode.getIdentifier(); - labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier); - checkAuths(auths, userName, labelOrdinal, identifier); - labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal. - } - if (labelOrdinal == 0) { - throw new InvalidLabelException("Invalid visibility label " + identifier); - } - labelOrdinals.add(labelOrdinal); - } else { - List childExps = ((NonLeafExpressionNode) node).getChildExps(); - for (ExpressionNode child : childExps) { - getLabelOrdinals(child, labelOrdinals, auths, userName); - } - } - } - - private void checkAuths(Set auths, String userName, int labelOrdinal, String identifier) - throws IOException { - if (checkAuths && !isSystemOrSuperUser()) { - if (auths == null || (!auths.contains(labelOrdinal))) { - throw new AccessDeniedException("Visibility label " + identifier - + " not authorized for the user " + userName); - } - } - } - @Override public RegionScanner preScannerOpen(ObserverContext e, Scan scan, RegionScanner s) throws IOException { + if (!initialized) throw new IOException("VisibilityController not yet initialized!!"); HRegion region = e.getEnvironment().getRegion(); Authorizations authorizations = null; - // If a super user issues a scan, he should be able to scan the cells - // irrespective of the Visibility labels - if (checkIfScanOrGetFromSuperUser()) { - return s; - } try { authorizations = scan.getAuthorizations(); } catch (DeserializationException de) { throw new IOException(de); } - Filter visibilityLabelFilter = createVisibilityLabelFilter(region, authorizations); + if (authorizations == null) { + // No Authorizations present for this scan/Get! + // In case of system tables other than "labels" just scan with out visibility check and + // filtering. Checking visibility labels for META and NAMESPACE table is not needed. + TableName table = region.getRegionInfo().getTable(); + if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) { + return s; + } + } + + Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(region, + authorizations, this.visibilityLabelService); if (visibilityLabelFilter != null) { Filter filter = scan.getFilter(); if (filter != null) { @@ -1039,15 +763,6 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb return s; } - private boolean checkIfScanOrGetFromSuperUser() throws IOException { - User user = getActiveUser(); - if (user != null && user.getShortName() != null) { - List auths = this.visibilityManager.getAuths(user.getShortName()); - return (auths.contains(SYSTEM_LABEL)); - } - return false; - } - @Override public DeleteTracker postInstantiateDeleteTracker( ObserverContext ctx, DeleteTracker delTracker) @@ -1064,10 +779,11 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb // but also on the visibility expression matching. return new VisibilityScanDeleteTracker(); } + @Override public RegionScanner postScannerOpen(final ObserverContext c, final Scan scan, final RegionScanner s) throws IOException { - User user = getActiveUser(); + User user = VisibilityUtils.getActiveUser(); if (user != null && user.getShortName() != null) { scannerOwners.put(s, user.getShortName()); } @@ -1112,90 +828,33 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb @Override public void preGetOp(ObserverContext e, Get get, List results) throws IOException { + if (!initialized) throw new IOException("VisibilityController not yet initialized!!"); + HRegion region = e.getEnvironment().getRegion(); Authorizations authorizations = null; - // If a super user issues a get, he should be able to scan the cells - // irrespective of the Visibility labels - if (checkIfScanOrGetFromSuperUser()) { - return; - } try { authorizations = get.getAuthorizations(); } catch (DeserializationException de) { throw new IOException(de); } - Filter visibilityLabelFilter = createVisibilityLabelFilter(e.getEnvironment().getRegion(), - authorizations); - if (visibilityLabelFilter != null) { - Filter filter = get.getFilter(); - if (filter != null) { - get.setFilter(new FilterList(filter, visibilityLabelFilter)); - } else { - get.setFilter(visibilityLabelFilter); - } - } - } - - private Filter createVisibilityLabelFilter(HRegion region, Authorizations authorizations) - throws IOException { - Map cfVsMaxVersions = new HashMap(); - for (HColumnDescriptor hcd : region.getTableDesc().getFamilies()) { - cfVsMaxVersions.put(new SimpleMutableByteRange(hcd.getName()), hcd.getMaxVersions()); - } if (authorizations == null) { // No Authorizations present for this scan/Get! // In case of system tables other than "labels" just scan with out visibility check and // filtering. Checking visibility labels for META and NAMESPACE table is not needed. TableName table = region.getRegionInfo().getTable(); if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) { - return null; - } - } else { - for (String label : authorizations.getLabels()) { - if (!VisibilityLabelsValidator.isValidLabel(label)) { - throw new IllegalArgumentException("Invalid authorization label : " + label - + ". Authorizations cannot contain '(', ')' ,'&' ,'|', '!'" + " and cannot be empty"); - } + return; } } - Filter visibilityLabelFilter = null; - if (this.scanLabelGenerators != null) { - List labels = null; - for (ScanLabelGenerator scanLabelGenerator : this.scanLabelGenerators) { - try { - // null authorizations to be handled inside SLG impl. - labels = scanLabelGenerator.getLabels(getActiveUser(), authorizations); - labels = (labels == null) ? new ArrayList() : labels; - authorizations = new Authorizations(labels); - } catch (Throwable t) { - LOG.error(t); - throw new IOException(t); - } - } - int labelsCount = this.visibilityManager.getLabelsCount(); - BitSet bs = new BitSet(labelsCount + 1); // ordinal is index 1 based - if (labels != null) { - for (String label : labels) { - int labelOrdinal = this.visibilityManager.getLabelOrdinal(label); - if (labelOrdinal != 0) { - bs.set(labelOrdinal); - } - } + Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(e.getEnvironment() + .getRegion(), authorizations, this.visibilityLabelService); + if (visibilityLabelFilter != null) { + Filter filter = get.getFilter(); + if (filter != null) { + get.setFilter(new FilterList(filter, visibilityLabelFilter)); + } else { + get.setFilter(visibilityLabelFilter); } - visibilityLabelFilter = new VisibilityLabelFilter(bs, cfVsMaxVersions); } - return visibilityLabelFilter; - } - - private User getActiveUser() throws IOException { - User user = RequestContext.getRequestUser(); - if (!RequestContext.isInRequestContext()) { - // for non-rpc handling, fallback to system user - user = User.getCurrent(); - } - if (LOG.isTraceEnabled()) { - LOG.trace("Current active user name is "+user.getShortName()); - } - return user; } private List getSystemAndSuperUsers() throws IOException { @@ -1214,7 +873,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } private boolean isSystemOrSuperUser() throws IOException { - User activeUser = getActiveUser(); + User activeUser = VisibilityUtils.getActiveUser(); return this.superUsers.contains(activeUser.getShortName()); } @@ -1253,27 +912,20 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (cellVisibility == null) { return newCell; } - Set auths = null; - User user = getActiveUser(); - if (checkAuths && user != null && user.getShortName() != null) { - auths = this.visibilityManager.getAuthsAsOrdinals(user.getShortName()); - } // Adding all other tags Iterator tagsItr = CellUtil.tagsIterator(newCell.getTagsArray(), newCell.getTagsOffset(), newCell.getTagsLength()); while (tagsItr.hasNext()) { Tag tag = tagsItr.next(); - if (tag.getType() != VisibilityUtils.VISIBILITY_TAG_TYPE - && tag.getType() != VisibilityUtils.VISIBILITY_EXP_SERIALIZATION_TAG_TYPE) { + if (tag.getType() != TagType.VISIBILITY_TAG_TYPE + && tag.getType() != TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { tags.add(tag); } } - try { - tags.addAll(createVisibilityTags(cellVisibility.getExpression(), true, auths, - user.getShortName())); - } catch (ParseException e) { - throw new IOException(e); - } + // Don't check user auths for labels with Mutations when the user is super user + boolean authCheck = this.checkAuths && !(isSystemOrSuperUser()); + tags.addAll(this.visibilityLabelService.createVisibilityExpTags(cellVisibility.getExpression(), + true, authCheck)); // We need to create another KV, unfortunately, because the current new KV // has no space for tags @@ -1298,61 +950,40 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb public synchronized void addLabels(RpcController controller, VisibilityLabelsRequest request, RpcCallback done) { VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder(); - List labels = request.getVisLabelList(); + List visLabels = request.getVisLabelList(); if (!initialized) { - setExceptionResults(labels.size(), new CoprocessorException( + setExceptionResults(visLabels.size(), new CoprocessorException( "VisibilityController not yet initialized"), response); - } - try { - checkCallingUserAuth(); - List puts = new ArrayList(labels.size()); - RegionActionResult successResult = RegionActionResult.newBuilder().build(); - for (VisibilityLabel visLabel : labels) { - byte[] label = visLabel.getLabel().toByteArray(); - String labelStr = Bytes.toString(label); - if (VisibilityLabelsValidator.isValidLabel(label)) { - if (this.visibilityManager.getLabelOrdinal(labelStr) > 0) { - RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); - failureResultBuilder.setException(ResponseConverter - .buildException(new LabelAlreadyExistsException("Label '" + labelStr - + "' already exists"))); - response.addResult(failureResultBuilder.build()); - } else { - Put p = new Put(Bytes.toBytes(ordinalCounter)); - p.addImmutable( - LABELS_TABLE_FAMILY, LABEL_QUALIFIER, label, LABELS_TABLE_TAGS); - if (LOG.isDebugEnabled()) { - LOG.debug("Adding the label "+labelStr); - } - puts.add(p); - ordinalCounter++; - response.addResult(successResult); - } - } else { - RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); - failureResultBuilder.setException(ResponseConverter - .buildException(new InvalidLabelException("Invalid visibility label '" + labelStr - + "'"))); - response.addResult(failureResultBuilder.build()); + } else { + try { + checkCallingUserAuth(); + List labels = new ArrayList(visLabels.size()); + RegionActionResult successResult = RegionActionResult.newBuilder().build(); + for (VisibilityLabel visLabel : visLabels) { + byte[] label = visLabel.getLabel().toByteArray(); + labels.add(label); + response.addResult(successResult); // Just mark as success. Later it will get reset + // based on the result from + // visibilityLabelService.addLabels () } - } - OperationStatus[] opStatus = this.regionEnv.getRegion().batchMutate( - puts.toArray(new Mutation[puts.size()])); - int i = 0; - for (OperationStatus status : opStatus) { - if (status.getOperationStatusCode() != SUCCESS) { - while (response.getResult(i) != successResult) + if (!labels.isEmpty()) { + OperationStatus[] opStatus = this.visibilityLabelService.addLabels(labels); + int i = 0; + for (OperationStatus status : opStatus) { + while (response.getResult(i) != successResult) i++; + if (status.getOperationStatusCode() != SUCCESS) { + RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); + failureResultBuilder.setException(ResponseConverter + .buildException(new DoNotRetryIOException(status.getExceptionMsg()))); + response.setResult(i, failureResultBuilder.build()); + } i++; - RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); - failureResultBuilder.setException(ResponseConverter - .buildException(new DoNotRetryIOException(status.getExceptionMsg()))); - response.setResult(i, failureResultBuilder.build()); + } } - i++; + } catch (IOException e) { + LOG.error(e); + setExceptionResults(visLabels.size(), e, response); } - } catch (IOException e) { - LOG.error(e); - setExceptionResults(labels.size(), e, response); } done.run(response.build()); } @@ -1367,34 +998,6 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } } - private void performACLCheck() throws IOException { - // Do ACL check only when the security is enabled. - if (this.acOn && !isSystemOrSuperUser()) { - User user = getActiveUser(); - throw new AccessDeniedException("User '" + (user != null ? user.getShortName() : "null") - + " is not authorized to perform this action."); - } - } - - private List> getExistingLabelsWithAuths() throws IOException { - Scan scan = new Scan(); - RegionScanner scanner = this.regionEnv.getRegion().getScanner(scan); - List> existingLabels = new ArrayList>(); - try { - while (true) { - List cells = new ArrayList(); - scanner.next(cells); - if (cells.isEmpty()) { - break; - } - existingLabels.add(cells); - } - } finally { - scanner.close(); - } - return existingLabels; - } - @Override public synchronized void setAuths(RpcController controller, SetAuthsRequest request, RpcCallback done) { @@ -1403,46 +1006,30 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (!initialized) { setExceptionResults(auths.size(), new CoprocessorException( "VisibilityController not yet initialized"), response); - } - byte[] user = request.getUser().toByteArray(); - try { - checkCallingUserAuth(); - List puts = new ArrayList(auths.size()); - RegionActionResult successResult = RegionActionResult.newBuilder().build(); - for (ByteString authBS : auths) { - byte[] auth = authBS.toByteArray(); - String authStr = Bytes.toString(auth); - int labelOrdinal = this.visibilityManager.getLabelOrdinal(authStr); - if (labelOrdinal == 0) { - // This label is not yet added. 1st this should be added to the system - RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); - failureResultBuilder.setException(ResponseConverter - .buildException(new InvalidLabelException("Label '" + authStr + "' doesn't exist"))); - response.addResult(failureResultBuilder.build()); - } else { - Put p = new Put(Bytes.toBytes(labelOrdinal)); - p.addImmutable( - LABELS_TABLE_FAMILY, user, DUMMY_VALUE, LABELS_TABLE_TAGS); - puts.add(p); - response.addResult(successResult); + } else { + try { + checkCallingUserAuth(); + List labelAuths = new ArrayList(auths.size()); + for (ByteString authBS : auths) { + labelAuths.add(authBS.toByteArray()); } - } - OperationStatus[] opStatus = this.regionEnv.getRegion().batchMutate( - puts.toArray(new Mutation[puts.size()])); - int i = 0; - for (OperationStatus status : opStatus) { - if (status.getOperationStatusCode() != SUCCESS) { - while (response.getResult(i) != successResult) i++; - RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); - failureResultBuilder.setException(ResponseConverter - .buildException(new DoNotRetryIOException(status.getExceptionMsg()))); - response.setResult(i, failureResultBuilder.build()); + OperationStatus[] opStatus = this.visibilityLabelService.setAuths(request.getUser() + .toByteArray(), labelAuths); + RegionActionResult successResult = RegionActionResult.newBuilder().build(); + for (OperationStatus status : opStatus) { + if (status.getOperationStatusCode() == SUCCESS) { + response.addResult(successResult); + } else { + RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); + failureResultBuilder.setException(ResponseConverter + .buildException(new DoNotRetryIOException(status.getExceptionMsg()))); + response.addResult(failureResultBuilder.build()); + } } - i++; + } catch (IOException e) { + LOG.error(e); + setExceptionResults(auths.size(), e, response); } - } catch (IOException e) { - LOG.error(e); - setExceptionResults(auths.size(), e, response); } done.run(response.build()); } @@ -1450,44 +1037,33 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb @Override public synchronized void getAuths(RpcController controller, GetAuthsRequest request, RpcCallback done) { - byte[] user = request.getUser().toByteArray(); GetAuthsResponse.Builder response = GetAuthsResponse.newBuilder(); - response.setUser(request.getUser()); - try { - List labels = getUserAuthsFromLabelsTable(user); - for (String label : labels) { - response.addAuth(ByteStringer.wrap(Bytes.toBytes(label))); + if (!initialized) { + controller.setFailed("VisibilityController not yet initialized"); + } else { + byte[] user = request.getUser().toByteArray(); + List labels = null; + try { + // We do ACL check here as we create scanner directly on region. It will not make calls to + // AccessController CP methods. + if (this.acOn && !isSystemOrSuperUser()) { + User requestingUser = VisibilityUtils.getActiveUser(); + throw new AccessDeniedException("User '" + + (requestingUser != null ? requestingUser.getShortName() : "null") + + " is not authorized to perform this action."); + } + labels = this.visibilityLabelService.getAuths(user, false); + } catch (IOException e) { + ResponseConverter.setControllerException(controller, e); } - } catch (IOException e) { - ResponseConverter.setControllerException(controller, e); - } - done.run(response.build()); - } - - private List getUserAuthsFromLabelsTable(byte[] user) throws IOException { - Scan s = new Scan(); - s.addColumn(LABELS_TABLE_FAMILY, user); - Filter filter = createVisibilityLabelFilter(this.regionEnv.getRegion(), new Authorizations( - SYSTEM_LABEL)); - s.setFilter(filter); - List auths = new ArrayList(); - // We do ACL check here as we create scanner directly on region. It will not make calls to - // AccessController CP methods. - performACLCheck(); - RegionScanner scanner = this.regionEnv.getRegion().getScanner(s); - List results = new ArrayList(1); - while (true) { - scanner.next(results); - if (results.isEmpty()) break; - Cell cell = results.get(0); - int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); - String label = this.visibilityManager.getLabel(ordinal); - if (label != null) { - auths.add(label); + response.setUser(request.getUser()); + if (labels != null) { + for (String label : labels) { + response.addAuth(ByteStringer.wrap(Bytes.toBytes(label))); + } } - results.clear(); } - return auths; + done.run(response.build()); } @Override @@ -1498,84 +1074,73 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (!initialized) { setExceptionResults(auths.size(), new CoprocessorException( "VisibilityController not yet initialized"), response); - } - byte[] user = request.getUser().toByteArray(); - try { - checkCallingUserAuth(); - List currentAuths = this.getUserAuthsFromLabelsTable(user); - List deletes = new ArrayList(auths.size()); - RegionActionResult successResult = RegionActionResult.newBuilder().build(); - for (ByteString authBS : auths) { - byte[] auth = authBS.toByteArray(); - String authStr = Bytes.toString(auth); - if (currentAuths.contains(authStr)) { - int labelOrdinal = this.visibilityManager.getLabelOrdinal(authStr); - assert labelOrdinal > 0; - Delete d = new Delete(Bytes.toBytes(labelOrdinal)); - d.deleteColumns(LABELS_TABLE_FAMILY, user); - deletes.add(d); - response.addResult(successResult); - } else { - // This label is not set for the user. - RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); - failureResultBuilder.setException(ResponseConverter - .buildException(new InvalidLabelException("Label '" + authStr - + "' is not set for the user " + Bytes.toString(user)))); - response.addResult(failureResultBuilder.build()); + } else { + try { + // When AC is ON, do AC based user auth check + if (this.acOn && !isSystemOrSuperUser()) { + User user = VisibilityUtils.getActiveUser(); + throw new AccessDeniedException("User '" + (user != null ? user.getShortName() : "null") + + " is not authorized to perform this action."); } - } - OperationStatus[] opStatus = this.regionEnv.getRegion().batchMutate( - deletes.toArray(new Mutation[deletes.size()])); - int i = 0; - for (OperationStatus status : opStatus) { - if (status.getOperationStatusCode() != SUCCESS) { - while (response.getResult(i) != successResult) i++; - RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); - failureResultBuilder.setException(ResponseConverter - .buildException(new DoNotRetryIOException(status.getExceptionMsg()))); - response.setResult(i, failureResultBuilder.build()); + checkCallingUserAuth(); // When AC is not in place the calling user should have SYSTEM_LABEL + // auth to do this action. + List labelAuths = new ArrayList(auths.size()); + for (ByteString authBS : auths) { + labelAuths.add(authBS.toByteArray()); + } + OperationStatus[] opStatus = this.visibilityLabelService.clearAuths(request.getUser() + .toByteArray(), labelAuths); + RegionActionResult successResult = RegionActionResult.newBuilder().build(); + for (OperationStatus status : opStatus) { + if (status.getOperationStatusCode() == SUCCESS) { + response.addResult(successResult); + } else { + RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder(); + failureResultBuilder.setException(ResponseConverter + .buildException(new DoNotRetryIOException(status.getExceptionMsg()))); + response.addResult(failureResultBuilder.build()); + } } - i++; + } catch (IOException e) { + LOG.error(e); + setExceptionResults(auths.size(), e, response); } - } catch (IOException e) { - LOG.error(e); - setExceptionResults(auths.size(), e, response); } done.run(response.build()); } private void checkCallingUserAuth() throws IOException { if (!this.acOn) { - User user = getActiveUser(); + User user = VisibilityUtils.getActiveUser(); if (user == null) { throw new IOException("Unable to retrieve calling user"); } - List auths = this.visibilityManager.getAuths(user.getShortName()); - if (LOG.isTraceEnabled()) { - LOG.trace("The list of auths are "+auths); - } - if (!auths.contains(SYSTEM_LABEL)) { + if (!(this.visibilityLabelService.havingSystemAuth(Bytes.toBytes(user.getShortName())))) { throw new AccessDeniedException("User '" + user.getShortName() + "' is not authorized to perform this action."); } } } - static class DeleteVersionVisibilityExpressionFilter extends FilterBase { - private List visibilityTags; + private static class DeleteVersionVisibilityExpressionFilter extends FilterBase { + private List deleteCellVisTags; + private Byte deleteCellVisTagsFormat; - public DeleteVersionVisibilityExpressionFilter(List visibilityTags) { - this.visibilityTags = visibilityTags; + public DeleteVersionVisibilityExpressionFilter(List deleteCellVisTags, + Byte deleteCellVisTagsFormat) { + this.deleteCellVisTags = deleteCellVisTags; + this.deleteCellVisTagsFormat = deleteCellVisTagsFormat; } @Override - public ReturnCode filterKeyValue(Cell kv) throws IOException { - boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(kv, visibilityTags); - if (matchFound) { - return ReturnCode.INCLUDE; - } else { - return ReturnCode.SKIP; - } + public ReturnCode filterKeyValue(Cell cell) throws IOException { + List putVisTags = new ArrayList(); + Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); + boolean matchFound = VisibilityLabelServiceManager + .getInstance().getVisibilityLabelService() + .matchVisibility(putVisTags, putCellVisTagsFormat, deleteCellVisTags, + deleteCellVisTagsFormat); + return matchFound ? ReturnCode.INCLUDE : ReturnCode.SKIP; } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityExpEvaluator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityExpEvaluator.java new file mode 100644 index 0000000..4a32fa7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityExpEvaluator.java @@ -0,0 +1,43 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; + +/** + * During the read (ie. get/Scan) the VisibilityController calls this interface for each of the + * Cell. Depending on the evaluate() result, a cell can be either included or filtered out from the + * read results. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface VisibilityExpEvaluator { + + /** + * Evaluates whether the passed cell passes Scan/Get Authorization. + * @param cell + * @return true if this cell can be included in the Result. Else false. + * @throws IOException + */ + boolean evaluate(Cell cell) throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java index 14d69ed..a080810 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java @@ -18,19 +18,13 @@ package org.apache.hadoop.hbase.security.visibility; import java.io.IOException; -import java.util.BitSet; -import java.util.Iterator; import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.filter.FilterBase; -import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.SimpleMutableByteRange; /** @@ -40,15 +34,16 @@ import org.apache.hadoop.hbase.util.SimpleMutableByteRange; @InterfaceAudience.Private class VisibilityLabelFilter extends FilterBase { - private final BitSet authLabels; + private final VisibilityExpEvaluator expEvaluator; private final Map cfVsMaxVersions; private final ByteRange curFamily; private final ByteRange curQualifier; private int curFamilyMaxVersions; private int curQualMetVersions; - public VisibilityLabelFilter(BitSet authLabels, Map cfVsMaxVersions) { - this.authLabels = authLabels; + public VisibilityLabelFilter(VisibilityExpEvaluator expEvaluator, + Map cfVsMaxVersions) { + this.expEvaluator = expEvaluator; this.cfVsMaxVersions = cfVsMaxVersions; this.curFamily = new SimpleMutableByteRange(); this.curQualifier = new SimpleMutableByteRange(); @@ -80,43 +75,7 @@ class VisibilityLabelFilter extends FilterBase { return ReturnCode.SKIP; } - Iterator tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), - cell.getTagsLength()); - boolean visibilityTagPresent = false; - while (tagsItr.hasNext()) { - boolean includeKV = true; - Tag tag = tagsItr.next(); - if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { - visibilityTagPresent = true; - int offset = tag.getTagOffset(); - int endOffset = offset + tag.getTagLength(); - while (offset < endOffset) { - Pair result = StreamUtils.readRawVarint32(tag.getBuffer(), offset); - int currLabelOrdinal = result.getFirst(); - if (currLabelOrdinal < 0) { - // check for the absence of this label in the Scan Auth labels - // ie. to check BitSet corresponding bit is 0 - int temp = -currLabelOrdinal; - if (this.authLabels.get(temp)) { - includeKV = false; - break; - } - } else { - if (!this.authLabels.get(currLabelOrdinal)) { - includeKV = false; - break; - } - } - offset += result.getSecond(); - } - if (includeKV) { - // We got one visibility expression getting evaluated to true. Good to include this KV in - // the result then. - return ReturnCode.INCLUDE; - } - } - } - return visibilityTagPresent ? ReturnCode.SKIP : ReturnCode.INCLUDE; + return this.expEvaluator.evaluate(cell) ? ReturnCode.INCLUDE : ReturnCode.SKIP; } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelService.java new file mode 100644 index 0000000..70c41b1 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelService.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.OperationStatus; + +/** + * The interface which deals with visibility labels and user auths admin service as well as the cell + * visibility expression storage part and read time evaluation. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface VisibilityLabelService extends Configurable { + + /** + * System calls this after opening of regions. Gives a chance for the VisibilityLabelService to so + * any initialization logic. + * @param e the region coprocessor env + */ + void init(RegionCoprocessorEnvironment e); + + /** + * Adds the set of labels into the system. + * @param labels + * @return OperationStatus[] + * @throws IOException + */ + OperationStatus[] addLabels(List labels) throws IOException; + + /** + * Sets given labels globally authorized for the user. + * @param user + * @param authLabels + * @return OperationStatus[] + * @throws IOException + */ + OperationStatus[] setAuths(byte[] user, List authLabels) throws IOException; + + /** + * Removes given labels from user's globally authorized list of labels. + * @param user + * @param authLabels + * @return OperationStatus[] + * @throws IOException + */ + OperationStatus[] clearAuths(byte[] user, List authLabels) throws IOException; + + /** + * @param user + * @param systemCall - whether a system or user originated call. + * @return Visibility labels authorized for the given user. + * @throws IOException + */ + List getAuths(byte[] user, boolean systemCall) throws IOException; + + /** + * @param visExpression + * @param withSerializationFormat specifies whether a tag, denoting the serialization version + * of the tags, to be added in the list. When this is true make sure to add the + * serialization format Tag also. The format tag value should be byte type. + * @param checkAuths denotes whether to check individual labels in visExpression against user's + * global auth label. + * @return The list of tags corresponds to the visibility expression. These tags will be stored + * along with the Cells. + * @throws IOException + */ + List createVisibilityExpTags(String visExpression, boolean withSerializationFormat, + boolean checkAuths) throws IOException; + + /** + * @param authorizations + * @return The VisibilityExpEvaluator corresponding to the given set of authorization labels. + * @throws IOException + */ + VisibilityExpEvaluator getVisibilityExpEvaluator(Authorizations authorizations) + throws IOException; + + /** + * @param user + * @return true if the given user is having system/super auth + * @throws IOException + */ + boolean havingSystemAuth(byte[] user) throws IOException; + + /** + * Checks for the matching visibility labels in the delete mutation and the cell in consideration + * @param putVisTags + * @param putTagsFormat + * @param deleteVisTags - that list of tags in the delete mutation (the specified Cell Visibility) + * @param deleteTagsFormat + * @return true if matching tags are found s + */ + boolean matchVisibility(List putVisTags, Byte putTagsFormat, List deleteVisTags, + Byte deleteTagsFormat) throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelServiceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelServiceManager.java new file mode 100644 index 0000000..02cd768 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelServiceManager.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * Manages singleton instance of {@link VisibilityLabelService} + */ +@InterfaceAudience.Private +public class VisibilityLabelServiceManager { + + public static final String VISIBILITY_LABEL_SERVICE_CLASS = + "hbase.regionserver.visibility.label.service.class"; + private static final VisibilityLabelServiceManager INSTANCE = new VisibilityLabelServiceManager(); + + private volatile VisibilityLabelService visibilityLabelService = null; + + private VisibilityLabelServiceManager() { + + } + + public static VisibilityLabelServiceManager getInstance() { + return INSTANCE; + } + + /** + * @param conf + * @return singleton instance of {@link VisibilityLabelService}. The FQCN of the implementation + * class can be specified using "hbase.regionserver.visibility.label.service.class". + */ + public VisibilityLabelService getVisibilityLabelService(Configuration conf) { + if (this.visibilityLabelService != null) { + return this.visibilityLabelService; + } + synchronized (this) { + if (this.visibilityLabelService != null) { + return this.visibilityLabelService; + } + Class clazz = conf.getClass(VISIBILITY_LABEL_SERVICE_CLASS, + DefaultVisibilityLabelServiceImpl.class, VisibilityLabelService.class); + this.visibilityLabelService = ReflectionUtils.newInstance(clazz, conf); + return this.visibilityLabelService; + } + } + + /** + * @return singleton instance of {@link VisibilityLabelService}. + * @throws IllegalStateException if this called before initialization of singleton instance. + */ + public VisibilityLabelService getVisibilityLabelService() { + // By the time this method is called, the singleton instance of visibilityLabelService should + // have been created. + if (this.visibilityLabelService == null) { + throw new IllegalStateException("VisibilityLabelService not yet instantiated"); + } + return this.visibilityLabelService; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsCache.java new file mode 100644 index 0000000..1171a43 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsCache.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations; +import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations; +import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; + +/** + * Maintains the cache for visibility labels and also uses the zookeeper to update the labels in the + * system. The cache updation happens based on the data change event that happens on the zookeeper + * znode for labels table + */ +@InterfaceAudience.Private +public class VisibilityLabelsCache { + + private static final Log LOG = LogFactory.getLog(VisibilityLabelsCache.class); + private static final List EMPTY_LIST = new ArrayList(0); + private static VisibilityLabelsCache instance; + + private ZKVisibilityLabelWatcher zkVisibilityWatcher; + private Map labels = new HashMap(); + private Map ordinalVsLabels = new HashMap(); + private Map> userAuths = new HashMap>(); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + + private VisibilityLabelsCache(ZooKeeperWatcher watcher, Configuration conf) throws IOException { + zkVisibilityWatcher = new ZKVisibilityLabelWatcher(watcher, this, conf); + try { + zkVisibilityWatcher.start(); + } catch (KeeperException ke) { + LOG.error("ZooKeeper initialization failed", ke); + throw new IOException(ke); + } + } + + public synchronized static VisibilityLabelsCache get(ZooKeeperWatcher watcher, + Configuration conf) throws IOException { + if (instance == null) { + instance = new VisibilityLabelsCache(watcher, conf); + } + return instance; + } + + public static VisibilityLabelsCache get() { + return instance; + } + + public void refreshLabelsCache(byte[] data) throws IOException { + List visibilityLabels = null; + try { + visibilityLabels = VisibilityUtils.readLabelsFromZKData(data); + } catch (DeserializationException dse) { + throw new IOException(dse); + } + this.lock.writeLock().lock(); + try { + for (VisibilityLabel visLabel : visibilityLabels) { + String label = Bytes.toString(visLabel.getLabel().toByteArray()); + labels.put(label, visLabel.getOrdinal()); + ordinalVsLabels.put(visLabel.getOrdinal(), label); + } + } finally { + this.lock.writeLock().unlock(); + } + } + + public void refreshUserAuthsCache(byte[] data) throws IOException { + MultiUserAuthorizations multiUserAuths = null; + try { + multiUserAuths = VisibilityUtils.readUserAuthsFromZKData(data); + } catch (DeserializationException dse) { + throw new IOException(dse); + } + this.lock.writeLock().lock(); + try { + for (UserAuthorizations userAuths : multiUserAuths.getUserAuthsList()) { + String user = Bytes.toString(userAuths.getUser().toByteArray()); + this.userAuths.put(user, new HashSet(userAuths.getAuthList())); + } + } finally { + this.lock.writeLock().unlock(); + } + } + + /** + * @param label + * @return The ordinal for the label. The ordinal starts from 1. Returns 0 when the passed a non + * existing label. + */ + public int getLabelOrdinal(String label) { + Integer ordinal = null; + this.lock.readLock().lock(); + try { + ordinal = labels.get(label); + } finally { + this.lock.readLock().unlock(); + } + if (ordinal != null) { + return ordinal.intValue(); + } + // 0 denotes not available + return 0; + } + + public String getLabel(int ordinal) { + this.lock.readLock().lock(); + try { + return this.ordinalVsLabels.get(ordinal); + } finally { + this.lock.readLock().unlock(); + } + } + + /** + * @return The total number of visibility labels. + */ + public int getLabelsCount(){ + return this.labels.size(); + } + + /** + * @param user + * @return The labels that the given user is authorized for. + */ + public List getAuths(String user) { + List auths = EMPTY_LIST; + this.lock.readLock().lock(); + try { + Set authOrdinals = userAuths.get(user); + if (authOrdinals != null) { + auths = new ArrayList(authOrdinals.size()); + for (Integer authOrdinal : authOrdinals) { + auths.add(ordinalVsLabels.get(authOrdinal)); + } + } + } finally { + this.lock.readLock().unlock(); + } + return auths; + } + + /** + * Returns the list of ordinals of authentications associated with the user + * + * @param user + * @return the list of ordinals + */ + public Set getAuthsAsOrdinals(String user) { + this.lock.readLock().lock(); + try { + return userAuths.get(user); + } finally { + this.lock.readLock().unlock(); + } + } + + /** + * Writes the labels data to zookeeper node. + * @param data + * @param labelsOrUserAuths true for writing labels and false for user auths. + */ + public void writeToZookeeper(byte[] data, boolean labelsOrUserAuths) { + this.zkVisibilityWatcher.writeToZookeeper(data, labelsOrUserAuths); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsManager.java deleted file mode 100644 index e840c64..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelsManager.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.security.visibility; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations; -import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations; -import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.zookeeper.KeeperException; - -/** - * Maintains the cache for visibility labels and also uses the zookeeper to update the labels in the - * system. The cache updation happens based on the data change event that happens on the zookeeper - * znode for labels table - */ -@InterfaceAudience.Private -public class VisibilityLabelsManager { - - private static final Log LOG = LogFactory.getLog(VisibilityLabelsManager.class); - private static final List EMPTY_LIST = new ArrayList(0); - private static VisibilityLabelsManager instance; - - private ZKVisibilityLabelWatcher zkVisibilityWatcher; - private Map labels = new HashMap(); - private Map ordinalVsLabels = new HashMap(); - private Map> userAuths = new HashMap>(); - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - - private VisibilityLabelsManager(ZooKeeperWatcher watcher, Configuration conf) throws IOException { - zkVisibilityWatcher = new ZKVisibilityLabelWatcher(watcher, this, conf); - try { - zkVisibilityWatcher.start(); - } catch (KeeperException ke) { - LOG.error("ZooKeeper initialization failed", ke); - throw new IOException(ke); - } - } - - public synchronized static VisibilityLabelsManager get(ZooKeeperWatcher watcher, - Configuration conf) throws IOException { - if (instance == null) { - instance = new VisibilityLabelsManager(watcher, conf); - } - return instance; - } - - public static VisibilityLabelsManager get() { - return instance; - } - - public void refreshLabelsCache(byte[] data) throws IOException { - List visibilityLabels = null; - try { - visibilityLabels = VisibilityUtils.readLabelsFromZKData(data); - } catch (DeserializationException dse) { - throw new IOException(dse); - } - this.lock.writeLock().lock(); - try { - for (VisibilityLabel visLabel : visibilityLabels) { - String label = Bytes.toString(visLabel.getLabel().toByteArray()); - labels.put(label, visLabel.getOrdinal()); - ordinalVsLabels.put(visLabel.getOrdinal(), label); - } - } finally { - this.lock.writeLock().unlock(); - } - } - - public void refreshUserAuthsCache(byte[] data) throws IOException { - MultiUserAuthorizations multiUserAuths = null; - try { - multiUserAuths = VisibilityUtils.readUserAuthsFromZKData(data); - } catch (DeserializationException dse) { - throw new IOException(dse); - } - this.lock.writeLock().lock(); - try { - for (UserAuthorizations userAuths : multiUserAuths.getUserAuthsList()) { - String user = Bytes.toString(userAuths.getUser().toByteArray()); - this.userAuths.put(user, new HashSet(userAuths.getAuthList())); - } - } finally { - this.lock.writeLock().unlock(); - } - } - - /** - * @param label - * @return The ordinal for the label. The ordinal starts from 1. Returns 0 when the passed a non - * existing label. - */ - public int getLabelOrdinal(String label) { - Integer ordinal = null; - this.lock.readLock().lock(); - try { - ordinal = labels.get(label); - } finally { - this.lock.readLock().unlock(); - } - if (ordinal != null) { - return ordinal.intValue(); - } - // 0 denotes not available - return 0; - } - - public String getLabel(int ordinal) { - this.lock.readLock().lock(); - try { - return this.ordinalVsLabels.get(ordinal); - } finally { - this.lock.readLock().unlock(); - } - } - - /** - * @return The total number of visibility labels. - */ - public int getLabelsCount(){ - return this.labels.size(); - } - - /** - * @param user - * @return The labels that the given user is authorized for. - */ - public List getAuths(String user) { - List auths = EMPTY_LIST; - this.lock.readLock().lock(); - try { - Set authOrdinals = userAuths.get(user); - if (authOrdinals != null) { - auths = new ArrayList(authOrdinals.size()); - for (Integer authOrdinal : authOrdinals) { - auths.add(ordinalVsLabels.get(authOrdinal)); - } - } - } finally { - this.lock.readLock().unlock(); - } - return auths; - } - - /** - * Returns the list of ordinals of authentications associated with the user - * - * @param user - * @return the list of ordinals - */ - public Set getAuthsAsOrdinals(String user) { - this.lock.readLock().lock(); - try { - return userAuths.get(user); - } finally { - this.lock.readLock().unlock(); - } - } - - /** - * Writes the labels data to zookeeper node. - * @param data - * @param labelsOrUserAuths true for writing labels and false for user auths. - */ - public void writeToZookeeper(byte[] data, boolean labelsOrUserAuths) { - this.zkVisibilityWatcher.writeToZookeeper(data, labelsOrUserAuths); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java index c151b74..4cdec2c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hbase.security.visibility; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -26,6 +27,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; @@ -33,6 +36,7 @@ import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; /** * Similar to ScanDeletTracker but tracks the visibility expression also before @@ -41,6 +45,8 @@ import org.apache.hadoop.hbase.util.Bytes; @InterfaceAudience.Private public class VisibilityScanDeleteTracker extends ScanDeleteTracker { + private static final Log LOG = LogFactory.getLog(VisibilityScanDeleteTracker.class); + // Its better to track the visibility tags in delete based on each type. Create individual // data structures for tracking each of them. This would ensure that there is no tracking based // on time and also would handle all cases where deletefamily or deletecolumns is specified with @@ -49,14 +55,17 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { // type would solve this problem and also ensure that the combination of different type // of deletes with diff ts would also work fine // Track per TS - private Map> visibilityTagsDeleteFamily = new HashMap>(); + private Map, Byte>> visibilityTagsDeleteFamily = + new HashMap, Byte>>(); // Delete family version with different ts and different visibility expression could come. // Need to track it per ts. - private Map> visibilityTagsDeleteFamilyVersion = new HashMap>(); - private List> visibilityTagsDeleteColumns; + private Map, Byte>> visibilityTagsDeleteFamilyVersion = + new HashMap, Byte>>(); + private List, Byte>> visibilityTagsDeleteColumns; // Tracking as List is to handle same ts cell but different visibility tag. // TODO : Need to handle puts with same ts but different vis tags. - private List> visiblityTagsDeleteColumnVersion = new ArrayList>(); + private List, Byte>> visiblityTagsDeleteColumnVersion = + new ArrayList, Byte>>(); public VisibilityScanDeleteTracker() { super(); @@ -110,61 +119,65 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { // If tag is present in the delete if (delCell.getTagsLength() > 0) { switch (type) { - case DeleteFamily: - List delTags = new ArrayList(); - if (visibilityTagsDeleteFamily != null) { - VisibilityUtils.getVisibilityTags(delCell, delTags); - if (!delTags.isEmpty()) { - visibilityTagsDeleteFamily.put(delCell.getTimestamp(), delTags); - } - } - break; - case DeleteFamilyVersion: - delTags = new ArrayList(); - VisibilityUtils.getVisibilityTags(delCell, delTags); - if (!delTags.isEmpty()) { - visibilityTagsDeleteFamilyVersion.put(delCell.getTimestamp(), delTags); - } - break; - case DeleteColumn: - if (visibilityTagsDeleteColumns == null) { - visibilityTagsDeleteColumns = new ArrayList>(); - } - delTags = new ArrayList(); - VisibilityUtils.getVisibilityTags(delCell, delTags); + case DeleteFamily: + List delTags = new ArrayList(); + if (visibilityTagsDeleteFamily != null) { + Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags); if (!delTags.isEmpty()) { - visibilityTagsDeleteColumns.add(delTags); + visibilityTagsDeleteFamily.put(delCell.getTimestamp(), new Pair, Byte>( + delTags, deleteCellVisTagsFormat)); } - break; - case Delete: - if (visiblityTagsDeleteColumnVersion == null) { - visiblityTagsDeleteColumnVersion = new ArrayList>(); - } - delTags = new ArrayList(); - VisibilityUtils.getVisibilityTags(delCell, delTags); - if (!delTags.isEmpty()) { - visiblityTagsDeleteColumnVersion.add(delTags); - } - break; - default: - throw new IllegalArgumentException("Invalid delete type"); + } + break; + case DeleteFamilyVersion: + delTags = new ArrayList(); + Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags); + if (!delTags.isEmpty()) { + visibilityTagsDeleteFamilyVersion.put(delCell.getTimestamp(), new Pair, Byte>( + delTags, deleteCellVisTagsFormat)); + } + break; + case DeleteColumn: + if (visibilityTagsDeleteColumns == null) { + visibilityTagsDeleteColumns = new ArrayList, Byte>>(); + } + delTags = new ArrayList(); + deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags); + if (!delTags.isEmpty()) { + visibilityTagsDeleteColumns.add(new Pair, Byte>(delTags, + deleteCellVisTagsFormat)); + } + break; + case Delete: + if (visiblityTagsDeleteColumnVersion == null) { + visiblityTagsDeleteColumnVersion = new ArrayList, Byte>>(); + } + delTags = new ArrayList(); + deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags); + if (!delTags.isEmpty()) { + visiblityTagsDeleteColumnVersion.add(new Pair, Byte>(delTags, + deleteCellVisTagsFormat)); + } + break; + default: + throw new IllegalArgumentException("Invalid delete type"); } } else { switch (type) { - case DeleteFamily: - visibilityTagsDeleteFamily = null; - break; - case DeleteFamilyVersion: - visibilityTagsDeleteFamilyVersion = null; - break; - case DeleteColumn: - visibilityTagsDeleteColumns = null; - break; - case Delete: - visiblityTagsDeleteColumnVersion = null; - break; - default: - throw new IllegalArgumentException("Invalid delete type"); + case DeleteFamily: + visibilityTagsDeleteFamily = null; + break; + case DeleteFamilyVersion: + visibilityTagsDeleteFamilyVersion = null; + break; + case DeleteColumn: + visibilityTagsDeleteColumns = null; + break; + case Delete: + visiblityTagsDeleteColumnVersion = null; + break; + default: + throw new IllegalArgumentException("Invalid delete type"); } } } @@ -174,93 +187,120 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { long timestamp = cell.getTimestamp(); int qualifierOffset = cell.getQualifierOffset(); int qualifierLength = cell.getQualifierLength(); - if (hasFamilyStamp) { - if (visibilityTagsDeleteFamily != null) { - Set>> deleteFamilies = visibilityTagsDeleteFamily.entrySet(); - Iterator>> iterator = deleteFamilies.iterator(); - while (iterator.hasNext()) { - Entry> entry = iterator.next(); - if (timestamp <= entry.getKey()) { - boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, - entry.getValue()); - if (matchFound) { - return DeleteResult.FAMILY_VERSION_DELETED; + try { + if (hasFamilyStamp) { + if (visibilityTagsDeleteFamily != null) { + Set, Byte>>> deleteFamilies = visibilityTagsDeleteFamily + .entrySet(); + Iterator, Byte>>> iterator = deleteFamilies.iterator(); + while (iterator.hasNext()) { + Entry, Byte>> entry = iterator.next(); + if (timestamp <= entry.getKey()) { + List putVisTags = new ArrayList(); + Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); + boolean matchFound = VisibilityLabelServiceManager + .getInstance() + .getVisibilityLabelService() + .matchVisibility(putVisTags, putCellVisTagsFormat, entry.getValue().getFirst(), + entry.getValue().getSecond()); + if (matchFound) { + return DeleteResult.FAMILY_VERSION_DELETED; + } } } - } - } else { - if (!VisibilityUtils.isVisibilityTagsPresent(cell)) { - // No tags - return DeleteResult.FAMILY_VERSION_DELETED; + } else { + if (!VisibilityUtils.isVisibilityTagsPresent(cell)) { + // No tags + return DeleteResult.FAMILY_VERSION_DELETED; + } } } - } - if (familyVersionStamps.contains(Long.valueOf(timestamp))) { - if (visibilityTagsDeleteFamilyVersion != null) { - List tags = visibilityTagsDeleteFamilyVersion.get(Long.valueOf(timestamp)); - if (tags != null) { - boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, tags); - if (matchFound) { + if (familyVersionStamps.contains(Long.valueOf(timestamp))) { + if (visibilityTagsDeleteFamilyVersion != null) { + Pair, Byte> tags = visibilityTagsDeleteFamilyVersion.get(Long + .valueOf(timestamp)); + if (tags != null) { + List putVisTags = new ArrayList(); + Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); + boolean matchFound = VisibilityLabelServiceManager + .getInstance() + .getVisibilityLabelService() + .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(), + tags.getSecond()); + if (matchFound) { + return DeleteResult.FAMILY_VERSION_DELETED; + } + } + } else { + if (!VisibilityUtils.isVisibilityTagsPresent(cell)) { + // No tags return DeleteResult.FAMILY_VERSION_DELETED; } } - } else { - if (!VisibilityUtils.isVisibilityTagsPresent(cell)) { - // No tags - return DeleteResult.FAMILY_VERSION_DELETED; - } } - } - if (deleteBuffer != null) { - int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, cell.getQualifierArray(), - qualifierOffset, qualifierLength); + if (deleteBuffer != null) { + int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, + cell.getQualifierArray(), qualifierOffset, qualifierLength); - if (ret == 0) { - if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { - if (visibilityTagsDeleteColumns != null) { - for (List tags : visibilityTagsDeleteColumns) { - boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, - tags); - if (matchFound) { + if (ret == 0) { + if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { + if (visibilityTagsDeleteColumns != null) { + for (Pair, Byte> tags : visibilityTagsDeleteColumns) { + List putVisTags = new ArrayList(); + Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); + boolean matchFound = VisibilityLabelServiceManager + .getInstance() + .getVisibilityLabelService() + .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(), + tags.getSecond()); + if (matchFound) { + return DeleteResult.VERSION_DELETED; + } + } + } else { + if (!VisibilityUtils.isVisibilityTagsPresent(cell)) { + // No tags return DeleteResult.VERSION_DELETED; } } - } else { - if (!VisibilityUtils.isVisibilityTagsPresent(cell)) { - // No tags - return DeleteResult.VERSION_DELETED; - } } - } - // Delete (aka DeleteVersion) - // If the timestamp is the same, keep this one - if (timestamp == deleteTimestamp) { - if (visiblityTagsDeleteColumnVersion != null) { - for (List tags : visiblityTagsDeleteColumnVersion) { - boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, - tags); - if (matchFound) { + // Delete (aka DeleteVersion) + // If the timestamp is the same, keep this one + if (timestamp == deleteTimestamp) { + if (visiblityTagsDeleteColumnVersion != null) { + for (Pair, Byte> tags : visiblityTagsDeleteColumnVersion) { + List putVisTags = new ArrayList(); + Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); + boolean matchFound = VisibilityLabelServiceManager + .getInstance() + .getVisibilityLabelService() + .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(), + tags.getSecond()); + if (matchFound) { + return DeleteResult.VERSION_DELETED; + } + } + } else { + if (!VisibilityUtils.isVisibilityTagsPresent(cell)) { + // No tags return DeleteResult.VERSION_DELETED; } } - } else { - if (!VisibilityUtils.isVisibilityTagsPresent(cell)) { - // No tags - return DeleteResult.VERSION_DELETED; - } } + } else if (ret < 0) { + // Next column case. + deleteBuffer = null; + visibilityTagsDeleteColumns = null; + visiblityTagsDeleteColumnVersion = null; + } else { + throw new IllegalStateException("isDeleted failed: deleteBuffer=" + + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier=" + + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength) + + ", timestamp=" + timestamp + ", comparison result: " + ret); } - } else if (ret < 0) { - // Next column case. - deleteBuffer = null; - visibilityTagsDeleteColumns = null; - visiblityTagsDeleteColumnVersion = null; - } else { - throw new IllegalStateException("isDeleted failed: deleteBuffer=" - + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier=" - + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength) - + ", timestamp=" + timestamp + ", comparison result: " + ret); } + } catch (IOException e) { + LOG.error("Error in isDeleted() check! Will treat cell as not deleted", e); } return DeleteResult.NOT_DELETED; } @@ -269,8 +309,8 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { public void reset() { super.reset(); visibilityTagsDeleteColumns = null; - visibilityTagsDeleteFamily = new HashMap>(); - visibilityTagsDeleteFamilyVersion = new HashMap>(); + visibilityTagsDeleteFamily = new HashMap, Byte>>(); + visibilityTagsDeleteFamilyVersion = new HashMap, Byte>>(); visiblityTagsDeleteColumnVersion = null; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java index 35843ff..2c2ea8e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java @@ -17,31 +17,40 @@ */ package org.apache.hadoop.hbase.security.visibility; +import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE; + import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.io.util.StreamUtils; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.ipc.RequestContext; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.SimpleMutableByteRange; import org.apache.hadoop.util.ReflectionUtils; import com.google.protobuf.InvalidProtocolBufferException; @@ -52,14 +61,14 @@ import com.google.protobuf.InvalidProtocolBufferException; @InterfaceAudience.Private public class VisibilityUtils { + private static final Log LOG = LogFactory.getLog(VisibilityUtils.class); + public static final String VISIBILITY_LABEL_GENERATOR_CLASS = "hbase.regionserver.scan.visibility.label.generator.class"; - public static final byte VISIBILITY_TAG_TYPE = TagType.VISIBILITY_TAG_TYPE; - public static final byte VISIBILITY_EXP_SERIALIZATION_TAG_TYPE = - TagType.VISIBILITY_EXP_SERIALIZATION_TAG_TYPE; public static final String SYSTEM_LABEL = "system"; - public static final Tag VIS_SERIALIZATION_TAG = new Tag(VISIBILITY_EXP_SERIALIZATION_TAG_TYPE, - VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT); + public static final Tag SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG = new Tag( + TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE, + VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG_VAL); private static final String COMMA = ","; /** @@ -140,8 +149,7 @@ public class VisibilityUtils { return null; } - public static List getScanLabelGenerators(Configuration conf) - throws IOException { + public static List getScanLabelGenerators(Configuration conf) { // There can be n SLG specified as comma separated in conf String slgClassesCommaSeparated = conf.get(VISIBILITY_LABEL_GENERATOR_CLASS); // We have only System level SLGs now. The order of execution will be same as the order in the @@ -155,7 +163,7 @@ public class VisibilityUtils { slgKlass = (Class) conf.getClassByName(slgClass.trim()); slgs.add(ReflectionUtils.newInstance(slgKlass, conf)); } catch (ClassNotFoundException e) { - throw new IOException(e); + throw new UnsupportedOperationException("Unable to find " + slgClass, e); } } } @@ -168,30 +176,24 @@ public class VisibilityUtils { } /** - * Get the list of visibility tags in the given cell + * Extract the visibility tags of the given Cell into the given List * @param cell - the cell - * @param tags - the tags array that will be populated if - * visibility tags are present - * @return true if the tags are in sorted order. + * @param tags - the array that will be populated if visibility tags are present + * @return The visibility tags serialization format */ - public static boolean getVisibilityTags(Cell cell, List tags) { - boolean sortedOrder = false; + public static Byte extractVisibilityTags(Cell cell, List tags) { + Byte serializationFormat = null; Iterator tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); - if (tag.getType() == VisibilityUtils.VISIBILITY_EXP_SERIALIZATION_TAG_TYPE) { - int serializationVersion = tag.getBuffer()[tag.getTagOffset()]; - if (serializationVersion == VisibilityConstants.VISIBILITY_SERIALIZATION_VERSION) { - sortedOrder = true; - continue; - } - } - if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { + if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { + serializationFormat = tag.getBuffer()[tag.getTagOffset()]; + } else if (tag.getType() == VISIBILITY_TAG_TYPE) { tags.add(tag); } } - return sortedOrder; + return serializationFormat; } /** @@ -204,124 +206,33 @@ public class VisibilityUtils { cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); - if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { + if (tag.getType() == VISIBILITY_TAG_TYPE) { return true; } } return false; } - /** - * Checks for the matching visibility labels in the delete mutation and - * the cell in consideration - * @param cell - the cell - * @param visibilityTagsInDeleteCell - that list of tags in the delete mutation - * (the specified Cell Visibility) - * @return true if matching tags are found - */ - public static boolean checkForMatchingVisibilityTags(Cell cell, - List visibilityTagsInDeleteCell) { - List tags = new ArrayList(); - boolean sortedTags = getVisibilityTags(cell, tags); - if (tags.size() == 0) { - // Early out if there are no tags in the cell - return false; - } - if (sortedTags) { - return checkForMatchingVisibilityTagsWithSortedOrder(visibilityTagsInDeleteCell, tags); - } else { - try { - return checkForMatchingVisibilityTagsWithOutSortedOrder(cell, visibilityTagsInDeleteCell); - } catch (IOException e) { - // Should not happen - throw new RuntimeException("Exception while sorting the tags from the cell", e); - } + public static Filter createVisibilityLabelFilter(HRegion region, Authorizations authorizations, + VisibilityLabelService service) throws IOException { + Map cfVsMaxVersions = new HashMap(); + for (HColumnDescriptor hcd : region.getTableDesc().getFamilies()) { + cfVsMaxVersions.put(new SimpleMutableByteRange(hcd.getName()), hcd.getMaxVersions()); } + Filter visibilityLabelFilter = new VisibilityLabelFilter( + service.getVisibilityExpEvaluator(authorizations), cfVsMaxVersions); + return visibilityLabelFilter; } - private static boolean checkForMatchingVisibilityTagsWithOutSortedOrder(Cell cell, - List visibilityTagsInDeleteCell) throws IOException { - List> sortedDeleteTags = sortTagsBasedOnOrdinal( - visibilityTagsInDeleteCell); - List> sortedTags = sortTagsBasedOnOrdinal(cell); - return compareTagsOrdinals(sortedDeleteTags, sortedTags); - } - - private static boolean checkForMatchingVisibilityTagsWithSortedOrder( - List visibilityTagsInDeleteCell, List tags) { - boolean matchFound = false; - if ((visibilityTagsInDeleteCell.size()) != tags.size()) { - // If the size does not match. Definitely we are not comparing the - // equal tags. - // Return false in that case. - return matchFound; + public static User getActiveUser() throws IOException { + User user = RequestContext.getRequestUser(); + if (!RequestContext.isInRequestContext()) { + // for non-rpc handling, fallback to system user + user = User.getCurrent(); } - for (Tag tag : visibilityTagsInDeleteCell) { - matchFound = false; - for (Tag givenTag : tags) { - if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(), - givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) { - matchFound = true; - break; - } - } - } - return matchFound; - } - - private static List> sortTagsBasedOnOrdinal(Cell cell) throws IOException { - Iterator tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), - cell.getTagsLength()); - List> fullTagsList = new ArrayList>(); - while (tagsItr.hasNext()) { - Tag tag = tagsItr.next(); - if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { - getSortedTagOrdinals(fullTagsList, tag); - } - } - return fullTagsList; - } - - private static List> sortTagsBasedOnOrdinal(List tags) throws IOException { - List> fullTagsList = new ArrayList>(); - for (Tag tag : tags) { - if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { - getSortedTagOrdinals(fullTagsList, tag); - } - } - return fullTagsList; - } - - private static void getSortedTagOrdinals(List> fullTagsList, Tag tag) - throws IOException { - List tagsOrdinalInSortedOrder = new ArrayList(); - int offset = tag.getTagOffset(); - int endOffset = offset + tag.getTagLength(); - while (offset < endOffset) { - Pair result = StreamUtils.readRawVarint32(tag.getBuffer(), offset); - tagsOrdinalInSortedOrder.add(result.getFirst()); - offset += result.getSecond(); - } - Collections.sort(tagsOrdinalInSortedOrder); - fullTagsList.add(tagsOrdinalInSortedOrder); - } - - private static boolean compareTagsOrdinals(List> tagsInDeletes, - List> tags) { - boolean matchFound = false; - if (tagsInDeletes.size() != tags.size()) { - return matchFound; - } else { - for (List deleteTagOrdinals : tagsInDeletes) { - matchFound = false; - for (List tagOrdinals : tags) { - if (deleteTagOrdinals.equals(tagOrdinals)) { - matchFound = true; - break; - } - } - } - return matchFound; + if (LOG.isTraceEnabled()) { + LOG.trace("Current active user name is " + user.getShortName()); } + return user; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ZKVisibilityLabelWatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ZKVisibilityLabelWatcher.java index 4f584e1..9a7f84a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ZKVisibilityLabelWatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ZKVisibilityLabelWatcher.java @@ -42,14 +42,14 @@ public class ZKVisibilityLabelWatcher extends ZooKeeperListener { "zookeeper.znode.visibility.user.auths.parent"; private static final String DEFAULT_VISIBILITY_USER_AUTHS_NODE = "visibility/user_auths"; - private VisibilityLabelsManager labelsManager; + private VisibilityLabelsCache labelsCache; private String labelZnode; private String userAuthsZnode; - public ZKVisibilityLabelWatcher(ZooKeeperWatcher watcher, VisibilityLabelsManager labelsManager, + public ZKVisibilityLabelWatcher(ZooKeeperWatcher watcher, VisibilityLabelsCache labelsManager, Configuration conf) { super(watcher); - this.labelsManager = labelsManager; + this.labelsCache = labelsManager; String labelZnodeParent = conf.get(VISIBILITY_LABEL_ZK_PATH, DEFAULT_VISIBILITY_LABEL_NODE); String userAuthsZnodeParent = conf.get(VISIBILITY_USER_AUTHS_ZK_PATH, DEFAULT_VISIBILITY_USER_AUTHS_NODE); @@ -75,7 +75,7 @@ public class ZKVisibilityLabelWatcher extends ZooKeeperListener { private void refreshVisibilityLabelsCache(byte[] data) { try { - this.labelsManager.refreshLabelsCache(data); + this.labelsCache.refreshLabelsCache(data); } catch (IOException ioe) { LOG.error("Failed parsing data from labels table " + " from zk", ioe); } @@ -83,7 +83,7 @@ public class ZKVisibilityLabelWatcher extends ZooKeeperListener { private void refreshUserAuthsCache(byte[] data) { try { - this.labelsManager.refreshUserAuthsCache(data); + this.labelsCache.refreshUserAuthsCache(data); } catch (IOException ioe) { LOG.error("Failed parsing data from labels table " + " from zk", ioe); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/ExpAsStringVisibilityLabelServiceImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/ExpAsStringVisibilityLabelServiceImpl.java new file mode 100644 index 0000000..84e886e --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/ExpAsStringVisibilityLabelServiceImpl.java @@ -0,0 +1,367 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE; +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY; +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; +import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.TagType; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.OperationStatus; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode; +import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode; +import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode; +import org.apache.hadoop.hbase.security.visibility.expression.Operator; +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.common.collect.Lists; + +@InterfaceAudience.Private +/** + * This is a VisibilityLabelService where labels in Mutation's visibility expression will be + * persisted as Strings itself rather than ordinals in 'labels' table. Also there is no need to add + * labels to the system, prior to using them in Mutations/Authorizations. + */ +public class ExpAsStringVisibilityLabelServiceImpl implements VisibilityLabelService { + + private static final Log LOG = LogFactory.getLog(ExpAsStringVisibilityLabelServiceImpl.class); + + private static final byte[] DUMMY_VALUE = new byte[0]; + private static final byte STRING_SERIALIZATION_FORMAT = 2; + private static final Tag STRING_SERIALIZATION_FORMAT_TAG = new Tag( + TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE, + new byte[]{STRING_SERIALIZATION_FORMAT}); + private final ExpressionParser expressionParser = new ExpressionParser(); + private final ExpressionExpander expressionExpander = new ExpressionExpander(); + private Configuration conf; + private HRegion labelsRegion; + + @Override + public OperationStatus[] addLabels(List labels) throws IOException { + // Not doing specific label add. We will just add labels in Mutation visibility expression as it + // is along with every cell. + OperationStatus[] status = new OperationStatus[labels.size()]; + for (int i = 0; i < labels.size(); i++) { + status[i] = new OperationStatus(OperationStatusCode.SUCCESS); + } + return status; + } + + @Override + public OperationStatus[] setAuths(byte[] user, List authLabels) throws IOException { + assert labelsRegion != null; + OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()]; + Put p = new Put(user); + for (byte[] auth : authLabels) { + p.add(LABELS_TABLE_FAMILY, auth, DUMMY_VALUE); + } + this.labelsRegion.put(p); + // This is a testing impl and so not doing any caching + for (int i = 0; i < authLabels.size(); i++) { + finalOpStatus[i] = new OperationStatus(OperationStatusCode.SUCCESS); + } + return finalOpStatus; + } + + @Override + public OperationStatus[] clearAuths(byte[] user, List authLabels) throws IOException { + assert labelsRegion != null; + OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()]; + List currentAuths = this.getAuths(user, true); + Delete d = new Delete(user); + int i = 0; + for (byte[] authLabel : authLabels) { + String authLabelStr = Bytes.toString(authLabel); + if (currentAuths.contains(authLabelStr)) { + d.deleteColumns(LABELS_TABLE_FAMILY, authLabel); + } else { + // This label is not set for the user. + finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE, + new InvalidLabelException("Label '" + authLabelStr + "' is not set for the user " + + Bytes.toString(user))); + } + i++; + } + this.labelsRegion.delete(d); + // This is a testing impl and so not doing any caching + for (i = 0; i < authLabels.size(); i++) { + if (finalOpStatus[i] == null) { + finalOpStatus[i] = new OperationStatus(OperationStatusCode.SUCCESS); + } + } + return finalOpStatus; + } + + @Override + public List getAuths(byte[] user, boolean systemCall) throws IOException { + assert (labelsRegion != null || systemCall); + List auths = new ArrayList(); + Get get = new Get(user); + List cells = null; + if (labelsRegion == null) { + HTable table = null; + try { + table = new HTable(conf, VisibilityConstants.LABELS_TABLE_NAME); + Result result = table.get(get); + cells = result.listCells(); + } finally { + if (table != null) { + table.close(); + } + } + } else { + cells = this.labelsRegion.get(get, false); + } + if (cells != null) { + for (Cell cell : cells) { + String auth = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), + cell.getQualifierLength()); + auths.add(auth); + } + } + return auths; + } + + @Override + public List createVisibilityExpTags(String visExpression, boolean withSerializationFormat, + boolean checkAuths) throws IOException { + ExpressionNode node = null; + try { + node = this.expressionParser.parse(visExpression); + } catch (ParseException e) { + throw new IOException(e); + } + node = this.expressionExpander.expand(node); + List tags = new ArrayList(); + if (withSerializationFormat) { + tags.add(STRING_SERIALIZATION_FORMAT_TAG); + } + if (node instanceof NonLeafExpressionNode + && ((NonLeafExpressionNode) node).getOperator() == Operator.OR) { + for (ExpressionNode child : ((NonLeafExpressionNode) node).getChildExps()) { + tags.add(createTag(child)); + } + } else { + tags.add(createTag(node)); + } + return tags; + } + + @Override + public VisibilityExpEvaluator getVisibilityExpEvaluator(final Authorizations authorizations) { + final List authLabels = authorizations == null ? new ArrayList() + : authorizations.getLabels(); + return new VisibilityExpEvaluator() { + @Override + public boolean evaluate(Cell cell) throws IOException { + Iterator tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), + cell.getTagsLength()); + boolean visibilityTagPresent = false; + while (tagsItr.hasNext()) { + boolean includeKV = true; + Tag tag = tagsItr.next(); + if (tag.getType() == VISIBILITY_TAG_TYPE) { + visibilityTagPresent = true; + int offset = tag.getTagOffset(); + int endOffset = offset + tag.getTagLength(); + while (offset < endOffset) { + short len = Bytes.toShort(tag.getBuffer(), offset); + offset += 2; + if (len < 0) { + // This is a NOT label. + len = (short) (-1 * len); + String label = Bytes.toString(tag.getBuffer(), offset, len); + if (authLabels.contains(label)) { + includeKV = false; + break; + } + } else { + String label = Bytes.toString(tag.getBuffer(), offset, len); + if (!authLabels.contains(label)) { + includeKV = false; + break; + } + } + offset += len; + } + if (includeKV) { + // We got one visibility expression getting evaluated to true. Good to include this KV + // in the result then. + return true; + } + } + } + return !(visibilityTagPresent); + } + }; + } + + private Tag createTag(ExpressionNode node) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + List labels = new ArrayList(); + List notLabels = new ArrayList(); + extractLabels(node, labels, notLabels); + Collections.sort(labels); + Collections.sort(notLabels); + // We will write the NOT labels 1st followed by normal labels + // Each of the label we will write with label length (as short 1st) followed by the label bytes. + // For a NOT node we will write the label length as -ve. + for (String label : notLabels) { + short length = (short) label.length(); + length = (short) (-1 * length); + dos.writeShort(length); + dos.write(Bytes.toBytes(label)); + } + for (String label : labels) { + dos.writeShort(label.length()); + dos.write(Bytes.toBytes(label)); + } + return new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()); + } + + private void extractLabels(ExpressionNode node, List labels, List notLabels) { + if (node.isSingleNode()) { + if (node instanceof NonLeafExpressionNode) { + // This is a NOT node. + LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node) + .getChildExps().get(0); + notLabels.add(lNode.getIdentifier()); + } else { + labels.add(((LeafExpressionNode) node).getIdentifier()); + } + } else { + // A non lead expression of labels with & operator. + NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node; + assert nlNode.getOperator() == Operator.AND; + List childExps = nlNode.getChildExps(); + for (ExpressionNode child : childExps) { + extractLabels(child, labels, notLabels); + } + } + } + + @Override + public Configuration getConf() { + return this.conf; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public void init(RegionCoprocessorEnvironment e) { + if (e.getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) { + this.labelsRegion = e.getRegion(); + // Set auth for "system" label for all super users. + try { + List superUsers = getSystemAndSuperUsers(); + for (String superUser : superUsers) { + byte[] user = Bytes.toBytes(superUser); + List auths = this.getAuths(user, true); + if (auths == null || auths.isEmpty()) { + Put p = new Put(user); + p.add(LABELS_TABLE_FAMILY, Bytes.toBytes(SYSTEM_LABEL), DUMMY_VALUE); + labelsRegion.put(p); + } + } + } catch (IOException ex) { + LOG.error(ex); + throw new RuntimeException(ex); + } + } + } + + private List getSystemAndSuperUsers() throws IOException { + User user = User.getCurrent(); + if (user == null) { + throw new IOException("Unable to obtain the current user, " + + "authorization checks for internal operations will not work correctly!"); + } + if (LOG.isTraceEnabled()) { + LOG.trace("Current user name is " + user.getShortName()); + } + String currentUser = user.getShortName(); + List superUsers = Lists.asList(currentUser, + this.conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0])); + return superUsers; + } + + @Override + public boolean havingSystemAuth(byte[] user) throws IOException { + List auths = this.getAuths(user, true); + return auths.contains(SYSTEM_LABEL); + } + + @Override + public boolean matchVisibility(List putTags, Byte putTagsFormat, List deleteTags, + Byte deleteTagsFormat) throws IOException { + assert putTagsFormat == STRING_SERIALIZATION_FORMAT; + assert deleteTagsFormat == STRING_SERIALIZATION_FORMAT; + return checkForMatchingVisibilityTagsWithSortedOrder(putTags, deleteTags); + } + + private static boolean checkForMatchingVisibilityTagsWithSortedOrder(List putVisTags, + List deleteVisTags) { + boolean matchFound = false; + if ((deleteVisTags.size()) != putVisTags.size()) { + // If the size does not match. Definitely we are not comparing the equal tags. + // Return false in that case. + return matchFound; + } + for (Tag tag : deleteVisTags) { + matchFound = false; + for (Tag givenTag : putVisTags) { + if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(), + givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) { + matchFound = true; + break; + } + } + } + return matchFound; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java index d5aa8a9..4e6406c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java @@ -95,7 +95,7 @@ public class TestVisibilityLabels { private volatile boolean killedRS = false; @Rule public final TestName TEST_NAME = new TestName(); - public static User SUPERUSER; + public static User SUPERUSER, USER1; @BeforeClass public static void setupBeforeClass() throws Exception { @@ -109,6 +109,7 @@ public class TestVisibilityLabels { conf.set("hbase.superuser", "admin"); TEST_UTIL.startMiniCluster(2); SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); + USER1 = User.createUserForTesting(conf, "user1", new String[] {}); // Wait for the labels table to become available TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000); @@ -299,7 +300,6 @@ public class TestVisibilityLabels { } }; t.start(); - regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads(); while (!killedRS) { Thread.sleep(10); } @@ -328,7 +328,7 @@ public class TestVisibilityLabels { s.setAuthorizations(new Authorizations(SECRET)); ResultScanner scanner = table.getScanner(s); Result[] next = scanner.next(3); - assertTrue(next.length == 1); + assertEquals(1, next.length); } finally { if (table != null) { table.close(); @@ -462,11 +462,19 @@ public class TestVisibilityLabels { List resultList = response.getResultList(); assertEquals(5, resultList.size()); assertTrue(resultList.get(0).getException().getValue().isEmpty()); - assertEquals("org.apache.hadoop.hbase.security.visibility.LabelAlreadyExistsException", - resultList.get(1).getException().getName()); + assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException", resultList.get(1) + .getException().getName()); + assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray()) + .contains( + "org.apache.hadoop.hbase.security.visibility.LabelAlreadyExistsException: " + + "Label 'secret' already exists")); assertTrue(resultList.get(2).getException().getValue().isEmpty()); - assertEquals("org.apache.hadoop.hbase.security.visibility.InvalidLabelException", - resultList.get(3).getException().getName()); + assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException", resultList.get(3) + .getException().getName()); + assertTrue(Bytes.toString(resultList.get(3).getException().getValue().toByteArray()) + .contains( + "org.apache.hadoop.hbase.security.visibility.InvalidLabelException: " + + "Invalid visibility label 'invalid~")); assertTrue(resultList.get(4).getException().getValue().isEmpty()); return null; } @@ -495,18 +503,14 @@ public class TestVisibilityLabels { scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL)); ResultScanner scanner = ht.getScanner(scan); Result result = null; + List results = new ArrayList(); while ((result = scanner.next()) != null) { - Cell label = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER); - Cell userAuth = result.getColumnLatestCell(LABELS_TABLE_FAMILY, user.getBytes()); - if (Bytes.equals(SECRET.getBytes(), 0, SECRET.getBytes().length, label.getValueArray(), - label.getValueOffset(), label.getValueLength()) - || Bytes.equals(CONFIDENTIAL.getBytes(), 0, CONFIDENTIAL.getBytes().length, - label.getValueArray(), label.getValueOffset(), label.getValueLength())) { - assertNotNull(userAuth); - } else { - assertNull(userAuth); - } + results.add(result); } + List auths = extractAuths(user, results); + assertTrue(auths.contains(SECRET)); + assertTrue(auths.contains(CONFIDENTIAL)); + assertEquals(2, auths.size()); } finally { if (ht != null) { ht.close(); @@ -560,6 +564,19 @@ public class TestVisibilityLabels { SUPERUSER.runAs(action); } + protected List extractAuths(String user, List results) { + List auths = new ArrayList(); + for (Result result : results) { + Cell labelCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER); + Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, user.getBytes()); + if (userAuthCell != null) { + auths.add(Bytes.toString(labelCell.getValueArray(), labelCell.getValueOffset(), + labelCell.getValueLength())); + } + } + return auths; + } + @Test public void testClearUserAuths() throws Throwable { PrivilegedExceptionAction action = new PrivilegedExceptionAction() { @@ -583,24 +600,25 @@ public class TestVisibilityLabels { List resultList = response.getResultList(); assertEquals(3, resultList.size()); assertTrue(resultList.get(0).getException().getValue().isEmpty()); - assertEquals("org.apache.hadoop.hbase.security.visibility.InvalidLabelException", + assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException", resultList.get(1).getException().getName()); + assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray()) + .contains( + "org.apache.hadoop.hbase.security.visibility.InvalidLabelException: " + + "Label 'public' is not set for the user testUser")); assertTrue(resultList.get(2).getException().getValue().isEmpty()); HTable ht = null; try { ht = new HTable(conf, LABELS_TABLE_NAME); ResultScanner scanner = ht.getScanner(new Scan()); Result result = null; + List results = new ArrayList(); while ((result = scanner.next()) != null) { - Cell label = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER); - Cell userAuth = result.getColumnLatestCell(LABELS_TABLE_FAMILY, user.getBytes()); - if (Bytes.equals(PRIVATE.getBytes(), 0, PRIVATE.getBytes().length, - label.getValueArray(), label.getValueOffset(), label.getValueLength())) { - assertNotNull(userAuth); - } else { - assertNull(userAuth); - } + results.add(result); } + List curAuths = extractAuths(user, results); + assertTrue(curAuths.contains(PRIVATE)); + assertEquals(1, curAuths.size()); } finally { if (ht != null) { ht.close(); @@ -901,7 +919,32 @@ public class TestVisibilityLabels { } } - private static HTable createTableAndWriteDataWithLabels(TableName tableName, String... labelExps) + @Test + public void testInvalidAuthorization() throws Exception { + final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTableDescriptor desc = new HTableDescriptor(tableName); + HColumnDescriptor col = new HColumnDescriptor(fam); + desc.addFamily(col); + TEST_UTIL.getHBaseAdmin().createTable(desc); + USER1.runAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); + try { + Get get = new Get(row1); + get.setAuthorizations(new Authorizations(CONFIDENTIAL + "&" + SECRET)); + table.get(get); + fail("Should have failed for invalid labels in Authorizations"); + } catch (IOException e) { + } finally { + table.close(); + } + return null; + } + }); + } + + static HTable createTableAndWriteDataWithLabels(TableName tableName, String... labelExps) throws Exception { HTable table = null; try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithCustomVisLabService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithCustomVisLabService.java new file mode 100644 index 0000000..bb4f18d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithCustomVisLabService.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY; +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; + +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableMap; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestVisibilityLabelsWithCustomVisLabService extends TestVisibilityLabels{ + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // setup configuration + conf = TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); + conf.setInt("hfile.format.version", 3); + conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName()); + conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName()); + conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, + ScanLabelGenerator.class); + conf.setClass(VisibilityLabelServiceManager.VISIBILITY_LABEL_SERVICE_CLASS, + ExpAsStringVisibilityLabelServiceImpl.class, VisibilityLabelService.class); + conf.set("hbase.superuser", "admin"); + TEST_UTIL.startMiniCluster(2); + SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); + + // Wait for the labels table to become available + TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000); + addLabels(); + } + + // Extending this test from super as we dont verify predefined labels in ExpAsStringVisibilityLabelServiceImpl + @Test + public void testVisibilityLabelsInPutsThatDoesNotMatchAnyDefinedLabels() throws Exception { + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + // This put with label "SAMPLE_LABEL" should not get failed. + createTableAndWriteDataWithLabels(tableName, "SAMPLE_LABEL", "TEST"); + } + + @Test + public void testAddVisibilityLabelsOnRSRestart() throws Exception { + // Do nothing + } + + @Test + public void testAddLabels() throws Exception { + // Do nothing + } + + @Test + public void testInvalidAuthorization() throws Exception { + // Do nothing + } + + protected List extractAuths(String user, List results) { + List auths = new ArrayList(); + for (Result result : results) { + if (Bytes.equals(result.getRow(), Bytes.toBytes(user))) { + NavigableMap familyMap = result.getFamilyMap(LABELS_TABLE_FAMILY); + for (byte[] q : familyMap.keySet()) { + auths.add(Bytes.toString(q, 0, q.length)); + } + } + } + return auths; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java index 3476f90..6ce4da2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java @@ -43,6 +43,7 @@ public class TestVisibilityLabelsWithDistributedLogReplay extends TestVisibility conf.set("hbase.superuser", "admin"); TEST_UTIL.startMiniCluster(2); SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); + USER1 = User.createUserForTesting(conf, "user1", new String[] {}); // Wait for the labels table to become available TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);