diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java index 24ce293..f89fe17 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java @@ -265,6 +265,32 @@ public class Delete extends Mutation implements Comparable { } /** + * Add the specified KeyValue to this Delete operation. Operation assumes that + * the passed Cell is immutable and its backing array will not be modified + * for the duration of this Delete. Internal usage only. + * @param kv individual Cell + * @return this + * @throws java.io.IOException e + */ + public Delete delete(Cell kv) throws IOException { + byte[] family = CellUtil.cloneFamily(kv); + List list = getCellList(family); + // Checking that the row of the kv is the same as the put + if (!KeyValue.isDelete(kv.getTypeByte())) { + throw new WrongRowIOException("The cell " + kv.toString() + "is not of Delete type"); + } + int res = Bytes.compareTo(this.row, 0, row.length, kv.getRowArray(), kv.getRowOffset(), + kv.getRowLength()); + if (res != 0) { + throw new WrongRowIOException("The row in " + kv.toString() + + " doesn't match the original one " + Bytes.toStringBinary(this.row)); + } + list.add(kv); + familyMap.put(family, list); + return this; + } + + /** * Delete the latest version of the specified column. * This is an expensive call in that on the server-side, it first does a * get to find the latest versions timestamp. Then it adds a delete using diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index 1ffb12b..b3c5f83 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -404,6 +404,14 @@ public final class CellUtil { return cell.getTypeByte() == Type.DeleteFamilyVersion.getCode(); } + public static boolean isDeleteColumns(final Cell cell) { + return cell.getTypeByte() == Type.DeleteColumn.getCode(); + } + + public static boolean isDeleteColumn(final Cell cell) { + return cell.getTypeByte() == Type.Delete.getCode(); + } + /** * @param cell * @return Estimate of the cell size in bytes. diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java index 96cc3bd..ef486ee 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -1111,8 +1111,8 @@ public interface RegionObserver extends Coprocessor { Cell postMutationBeforeWAL(ObserverContext ctx, MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException; - /** - * Called after the ScanQueryMatcher creates ScanDeleteTracker. Implementing + + /* Called after the ScanQueryMatcher creates ScanDeleteTracker. Implementing * this hook would help in creating customised DeleteTracker and returning * the newly created DeleteTracker * diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java index 0c133f6..3be8feb 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java @@ -85,19 +85,20 @@ public class LabelExpander { DataOutputStream dos = new DataOutputStream(baos); if (node.isSingleNode()) { writeLabelOrdinalsToStream(node, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, VisibilityUtils.appendPBuf(baos))); baos.reset(); } else { NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node; if (nlNode.getOperator() == Operator.OR) { for (ExpressionNode child : nlNode.getChildExps()) { writeLabelOrdinalsToStream(child, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, + VisibilityUtils.appendPBuf(baos))); baos.reset(); } } else { writeLabelOrdinalsToStream(nlNode, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, VisibilityUtils.appendPBuf(baos))); baos.reset(); } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java index 514628d..469d451 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java @@ -45,7 +45,7 @@ public interface DeleteTracker { /** * Check if the specified cell buffer has been deleted by a previously * seen delete. - * @param cell - current cell to check if deleted by a previously deleted cell + * @param cell - current cell to check if deleted by a previously seen delete * @return deleteResult The result tells whether the KeyValue is deleted and why */ DeleteResult isDeleted(Cell cell); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java index 6a767e8..668e3db 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java @@ -45,14 +45,14 @@ import org.apache.hadoop.hbase.util.Bytes; @InterfaceAudience.Private public class ScanDeleteTracker implements DeleteTracker { - private boolean hasFamilyStamp = false; - private long familyStamp = 0L; - private SortedSet familyVersionStamps = new TreeSet(); - private byte [] deleteBuffer = null; - private int deleteOffset = 0; - private int deleteLength = 0; - private byte deleteType = 0; - private long deleteTimestamp = 0L; + protected boolean hasFamilyStamp = false; + protected long familyStamp = 0L; + protected SortedSet familyVersionStamps = new TreeSet(); + protected byte [] deleteBuffer = null; + protected int deleteOffset = 0; + protected int deleteLength = 0; + protected byte deleteType = 0; + protected long deleteTimestamp = 0L; /** * Constructor for ScanDeleteTracker @@ -65,7 +65,7 @@ public class ScanDeleteTracker implements DeleteTracker { * Add the specified KeyValue to the list of deletes to check against for * this row operation. *

- * This is called when a Delete is encountered in a StoreFile. + * This is called when a Delete is encountered. * @param cell - the delete cell */ @Override diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java index 97c02b3..963a8fa 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.FilterBase; -import org.apache.hadoop.hbase.filter.Filter.ReturnCode; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index cccb3ff..c601a22 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -30,10 +30,12 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -57,6 +59,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; @@ -93,12 +96,14 @@ import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.Visibil import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.DeleteTracker; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; import org.apache.hadoop.hbase.regionserver.OperationStatus; import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessControlLists; @@ -656,10 +661,8 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb new OperationStatus(SANITY_CHECK_FAILURE, de.getMessage())); continue; } - if (m instanceof Put) { - Put p = (Put) m; boolean sanityFailure = false; - for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) { + for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, "Mutation contains cell with reserved type tag")); @@ -685,7 +688,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (visibilityTags != null) { labelCache.put(labelsExp, visibilityTags); List updatedCells = new ArrayList(); - for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) { + for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { Cell cell = cellScanner.current(); List tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); @@ -698,18 +701,107 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb cell.getValueLength(), tags); updatedCells.add(updatedCell); } - p.getFamilyCellMap().clear(); + m.getFamilyCellMap().clear(); // Clear and add new Cells to the Mutation. for (Cell cell : updatedCells) { - p.add(cell); + if(m instanceof Put) { + Put p = (Put)m; + p.add(cell); + } else if(m instanceof Delete) { + // Should we fail any delete without CellVisibility? + Delete d = (Delete)m; + d.delete(cell); + } } } } + } + } + } + + @Override + public void preDelete(ObserverContext env, Delete delete, + WALEdit edit, Durability durability) throws IOException { + // Here we have two cases + // Case 1: Delete column with specified ts + // If the cell's ts matches with the specified ts in the delete Column cell, + // we have found a match. + // Case 2 : Delete column with latest ts + // Now the issue here is, in the HRegion.batchMutate(), in + // prepareDeleteTimestamps() + // the timestamp for the delete is updated with the latest timestamp. + // If suppose the latest version of the cell does not have the required + // visibility expression + // we should go ahead with the next latest version of the cell where we + // could match the visibility + // expression. + CellVisibility cellVisibility = null; + try { + cellVisibility = delete.getCellVisibility(); + } catch (DeserializationException de) { + throw new RuntimeException("Invalid cell visibility specified " + delete); + } + for (CellScanner cellScanner = delete.cellScanner(); cellScanner.advance();) { + if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { + throw new RuntimeException("Reserved visibility type used " + cellVisibility.toString()); + } + } + List visibilityTags = new ArrayList(); + if (cellVisibility != null) { + String labelsExp = cellVisibility.getExpression(); + try { + visibilityTags = createVisibilityTags(labelsExp); + } catch (ParseException e) { + throw new RuntimeException("Invalid cell visibility expression " + labelsExp); + } catch (InvalidLabelException e) { + throw new RuntimeException("Invalid cell visibility specified " + labelsExp); + } + } + for (Map.Entry> e : delete.getFamilyCellMap().entrySet()) { + byte[] family = e.getKey(); + List cells = e.getValue(); + Map kvCount = new TreeMap(Bytes.BYTES_COMPARATOR); + boolean matchFound = false; + boolean isDeleteLatest = false; + for (Cell cell : cells) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + // Check if time is LATEST, change to time of most recent addition if so + // This is expensive. + if (kv.isLatestTimestamp() && kv.isDeleteType()) { + isDeleteLatest = true; + byte[] qual = kv.getQualifier(); + if (qual == null) + qual = HConstants.EMPTY_BYTE_ARRAY; + Get get = new Get(kv.getRow()); + // Fetch all the max versions possible + get.setMaxVersions(); + + get.addColumn(family, qual); + List results = new ArrayList(); + RegionScanner scanner = null; + try { + Scan scan = new Scan(get); + // scan.setRaw(true); + scanner = env.getEnvironment().getRegion().getScanner(scan); + scanner.next(results); + } finally { + if (scanner != null) + scanner.close(); + } + for (Cell getkv : results) { + matchFound = VisibilityUtils.checkForMatchingVisibilityTags(getkv, visibilityTags); + if (matchFound) { + KeyValue getKeyVal = KeyValueUtil.ensureKeyValue(getkv); + Bytes.putBytes(kv.getBuffer(), kv.getTimestampOffset(), getKeyVal.getBuffer(), + getKeyVal.getTimestampOffset(), Bytes.SIZEOF_LONG); + break; + } + } } - } else if (cellVisibility != null) { - // CellVisibility in a Delete is not legal! Fail the operation - miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, - "CellVisibility cannot be set on Delete mutation")); + } + if (!matchFound && isDeleteLatest) { +/* throw new RuntimeException("No matching cell with given visibility expression found " + + delete);*/ } } } @@ -820,28 +912,43 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb List tags = new ArrayList(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); + List labelOrdinals = new ArrayList(); + // Append PBUF magic byte to indicate that the latest puts from 0.98.2 onwards are sorted. if (node.isSingleNode()) { - writeLabelOrdinalsToStream(node, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + getLabelOrdinals(node, labelOrdinals); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, VisibilityUtils.appendPBuf(baos))); baos.reset(); } else { NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node; if (nlNode.getOperator() == Operator.OR) { for (ExpressionNode child : nlNode.getChildExps()) { - writeLabelOrdinalsToStream(child, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + getLabelOrdinals(child, labelOrdinals); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, + VisibilityUtils.appendPBuf(baos))); baos.reset(); + labelOrdinals.clear(); } } else { - writeLabelOrdinalsToStream(nlNode, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + getLabelOrdinals(nlNode, labelOrdinals); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, VisibilityUtils.appendPBuf(baos))); baos.reset(); } } return tags; } - private void writeLabelOrdinalsToStream(ExpressionNode node, DataOutputStream dos) + private void writeLabelOrdinalsToStream(List labelOrdinals, DataOutputStream dos) + throws IOException { + Collections.sort(labelOrdinals); + for (Integer labelOrdinal : labelOrdinals) { + StreamUtils.writeRawVInt32(dos, labelOrdinal); + } + } + + private void getLabelOrdinals(ExpressionNode node, List labelOrdinals) throws IOException, InvalidLabelException { if (node.isSingleNode()) { String identifier = null; @@ -864,11 +971,11 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (labelOrdinal == 0) { throw new InvalidLabelException("Invalid visibility label " + identifier); } - StreamUtils.writeRawVInt32(dos, labelOrdinal); + labelOrdinals.add(labelOrdinal); } else { List childExps = ((NonLeafExpressionNode) node).getChildExps(); for (ExpressionNode child : childExps) { - writeLabelOrdinalsToStream(child, dos); + getLabelOrdinals(child, labelOrdinals); } } } @@ -910,6 +1017,12 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } @Override + public DeleteTracker postInstantiateDeleteTracker( + ObserverContext ctx, DeleteTracker delTracker) + throws IOException { + return new VisibilityScanDeleteTracker(); + } + @Override public RegionScanner postScannerOpen(final ObserverContext c, final Scan scan, final RegionScanner s) throws IOException { User user = getActiveUser(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java index 0ff8d67..8e59325 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityLabelFilter.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.filter.FilterBase; import org.apache.hadoop.hbase.io.util.StreamUtils; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -46,6 +47,7 @@ class VisibilityLabelFilter extends FilterBase { private final ByteRange curQualifier; private int curFamilyMaxVersions; private int curQualMetVersions; + private static int protoBufLen = ProtobufUtil.lengthOfPBMagic(); public VisibilityLabelFilter(BitSet authLabels, Map cfVsMaxVersions) { this.authLabels = authLabels; @@ -90,6 +92,11 @@ class VisibilityLabelFilter extends FilterBase { visibilityTagPresent = true; int offset = tag.getTagOffset(); int endOffset = offset + tag.getTagLength(); + boolean pbMagicPrefix = ProtobufUtil.isPBMagicPrefix(tag.getBuffer(), offset, + tag.getTagLength()); + if(pbMagicPrefix) { + offset += protoBufLen; + } while (offset < endOffset) { Pair result = StreamUtils.readRawVarint32(tag.getBuffer(), offset); int currLabelOrdinal = result.getFirst(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java new file mode 100644 index 0000000..2ee0f34 --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java @@ -0,0 +1,202 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import java.util.List; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Similar to ScanDeletTracker but tracks the visibility expression also before + * deciding if a Cell can be considered deleted + */ +public class VisibilityScanDeleteTracker extends ScanDeleteTracker { + + private List visibilityTagsInDeleteCell; + + public VisibilityScanDeleteTracker() { + super(); + } + + @Override + public void add(Cell delCell) { + //Cannot call super.add because need to find if the delete needs to be considered + long timestamp = delCell.getTimestamp(); + int qualifierOffset = delCell.getQualifierOffset(); + int qualifierLength = delCell.getQualifierLength(); + byte type = delCell.getTypeByte(); + if (!hasFamilyStamp || timestamp > familyStamp) { + if (type == KeyValue.Type.DeleteFamily.getCode()) { + hasFamilyStamp = true; + familyStamp = timestamp; + extractDeleteTags(delCell); + return; + } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) { + familyVersionStamps.add(timestamp); + extractDeleteTags(delCell); + return; + } + + if (deleteBuffer != null && type < deleteType) { + // same column, so ignore less specific delete + if (Bytes.equals(deleteBuffer, deleteOffset, deleteLength, + delCell.getQualifierArray(), qualifierOffset, qualifierLength)){ + return; + } + } + // new column, or more general delete type + deleteBuffer = delCell.getQualifierArray(); + deleteOffset = qualifierOffset; + deleteLength = qualifierLength; + deleteType = type; + deleteTimestamp = timestamp; + extractDeleteTags(delCell); + } + } + + private void extractDeleteTags(Cell delCell) { + // If tag is present in the delete + if (delCell.getTagsLength() > 0) { + visibilityTagsInDeleteCell = VisibilityUtils.getTagsFromResult(delCell); + if (visibilityTagsInDeleteCell.isEmpty()) { + visibilityTagsInDeleteCell = null; + } + } else { + visibilityTagsInDeleteCell = null; + } + } + + @Override + public DeleteResult isDeleted(Cell cell) { + long timestamp = cell.getTimestamp(); + int qualifierOffset = cell.getQualifierOffset(); + int qualifierLength = cell.getQualifierLength(); + if (hasFamilyStamp && timestamp <= familyStamp) { + if (visibilityTagsInDeleteCell != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsInDeleteCell); + if (matchFound) { + return DeleteResult.FAMILY_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + } else { + List tags = VisibilityUtils.getTagsFromResult(cell); + if(tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.FAMILY_DELETED; + } + } + } + + if (familyVersionStamps.contains(Long.valueOf(timestamp))) { + if (visibilityTagsInDeleteCell != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsInDeleteCell); + if (matchFound) { + return DeleteResult.FAMILY_VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + } else { + List tags = VisibilityUtils.getTagsFromResult(cell); + if(tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.FAMILY_VERSION_DELETED; + } + } + } + + if (deleteBuffer != null) { + int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, cell.getQualifierArray(), + qualifierOffset, qualifierLength); + + if (ret == 0) { + if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { + if (visibilityTagsInDeleteCell != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsInDeleteCell); + if (matchFound) { + return DeleteResult.COLUMN_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + } else { + List tags = VisibilityUtils.getTagsFromResult(cell); + if(tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.COLUMN_DELETED; + } + } + } + // Delete (aka DeleteVersion) + // If the timestamp is the same, keep this one + if (timestamp == deleteTimestamp) { + if (visibilityTagsInDeleteCell != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsInDeleteCell); + if (matchFound) { + return DeleteResult.VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + } else { + List tags = VisibilityUtils.getTagsFromResult(cell); + if(tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.VERSION_DELETED; + } + } + } + // use assert or not? + assert timestamp < deleteTimestamp; + + // different timestamp, let's clear the buffer. + deleteBuffer = null; + } else if (ret < 0) { + // Next column case. + deleteBuffer = null; + } else { + throw new IllegalStateException("isDelete failed: deleteBuffer=" + + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier=" + + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength) + + ", timestamp=" + timestamp + ", comparison result: " + ret); + } + } + + return DeleteResult.NOT_DELETED; + } + + // For BC related checks, once we know that the byte[] does not have the magic + // byte try using BitSets. for the visibility tags in the delete set the corresponding bits + // where there is a tag. NOT condition should also be handled. check whether the same bits + // are set in the labels in the Puts. If so then say matchFound = true. +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java index 0c7764f..9f9973d 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java @@ -17,27 +17,35 @@ */ package org.apache.hadoop.hbase.security.visibility; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import com.google.protobuf.HBaseZeroCopyByteString; - import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.util.ReflectionUtils; +import com.google.protobuf.HBaseZeroCopyByteString; import com.google.protobuf.InvalidProtocolBufferException; /** @@ -51,6 +59,7 @@ public class VisibilityUtils { public static final byte VISIBILITY_TAG_TYPE = TagType.VISIBILITY_TAG_TYPE; public static final String SYSTEM_LABEL = "system"; private static final String COMMA = ","; + private static final int PBUFLEN = ProtobufUtil.lengthOfPBMagic(); /** * Creates the labels data to be written to zookeeper. @@ -68,6 +77,16 @@ public class VisibilityUtils { return ProtobufUtil.prependPBMagic(visReqBuilder.build().toByteArray()); } + public static byte[] appendPBuf(ByteArrayOutputStream baos) { + // Creating a protobuf object is like during parse of every cell we would create + // those protobuf objects? Is it needed here? + + //VisibilityTag.Builder tagBuilder = VisibilityTag.newBuilder(); + //tagBuilder.setTag(HBaseZeroCopyByteString.wrap(baos.toByteArray())); + //return ProtobufUtil.prependPBMagic(tagBuilder.build().toByteArray()); + return ProtobufUtil.prependPBMagic(baos.toByteArray()); + } + /** * Creates the user auth data to be written to zookeeper. * @param userAuths @@ -156,4 +175,150 @@ public class VisibilityUtils { } return slgs; } + + public static List getTagsFromResult(Cell cell) { + List existingTags = new ArrayList(); + byte[] tags = CellUtil.getTagArray(cell); + Iterator tagsIterator = CellUtil.tagsIterator(tags, 0, tags.length); + while (tagsIterator.hasNext()) { + Tag tag = tagsIterator.next(); + if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { + existingTags.add(tag); + } + } + return existingTags; + } + + public static boolean checkForMatchingVisibilityTags(Cell cell, + List visibilityTagsInDeleteCell) { + List tags = getTagsFromResult(cell); + boolean pbMagicPrefix = true; + // From the first tag identify if it has PBUF. If yes then all the tags will + // be having PBUF + if (tags.size() > 0) { + pbMagicPrefix = VisibilityUtils.isPBMagicPrefixedTag(tags.get(0)); + } else { + // Early out if there are no tags in the cell + return false; + } + // Check for the first byte to know + if (pbMagicPrefix) { + return checkForMatchingVisibilityTagsWithPBUF(cell.getTagsLength(), + visibilityTagsInDeleteCell, tags); + } else { + try { + return checkForMatchingVisibilityTagsWithOutPBUF(cell, visibilityTagsInDeleteCell); + } catch (Exception e) { + // Should not happen + throw new RuntimeException("Exception while sorting the tags from the cell"); + } + } + } + + private static boolean checkForMatchingVisibilityTagsWithOutPBUF(Cell cell, + List visibilityTagsInDeleteCell) throws IOException { + List> sortedDeleteTags = getVisibilityTagsWithoutMagicBytes( + visibilityTagsInDeleteCell, true); + List> sortedTags = getVisibilityTagsWithoutMagicBytes(cell, false); + return compareTagsOrdinals(sortedDeleteTags, sortedTags); + } + + private static boolean checkForMatchingVisibilityTagsWithPBUF(short tagsLength, + List visibilityTagsInDeleteCell, List tags) { + boolean matchFound = false; + if (tagsLength > 0) { + if (visibilityTagsInDeleteCell.size() != tags.size()) { + // If the size does not match. Definitely we are not comparing the equal + // tags. + // Return false in that case. + return matchFound; + } + for (Tag tag : visibilityTagsInDeleteCell) { + // for the delete tags the PBUF will be there. + matchFound = false; + for (Tag givenTag : tags) { + if (Bytes.equals(tag.getBuffer(), tag.getTagOffset() + PBUFLEN, tag.getTagLength() + - PBUFLEN, givenTag.getBuffer(), givenTag.getTagOffset() + PBUFLEN, + givenTag.getTagLength() - PBUFLEN)) { + matchFound = true; + break; + } + } + } + } + return matchFound; + } + + public static boolean isPBMagicPrefixedTag(Tag tag) { + return ProtobufUtil.isPBMagicPrefix(tag.getBuffer(), + tag.getTagOffset(), tag.getTagLength()); + } + + private static List> getVisibilityTagsWithoutMagicBytes(Cell cell, + boolean pbMagicPrefix) throws IOException { + Iterator tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), + cell.getTagsLength()); + List> fullTagsList = new ArrayList>(); + List tagsOrdinalInSortedOrder = new ArrayList(); + while (tagsItr.hasNext()) { + Tag tag = tagsItr.next(); + getSortedTagOrdinals(fullTagsList, tagsOrdinalInSortedOrder, tag, pbMagicPrefix); + } + return fullTagsList; + } + + private static List> getVisibilityTagsWithoutMagicBytes(List tags, + boolean pbMagicPrefix) throws IOException { + // Currently assume there is one set of visibility tag per cell + // TODO : handle more than one visibility tag + List> fullTagsList = new ArrayList>(); + List tagsOrdinalInSortedOrder = new ArrayList(); + for (Tag tag : tags) { + getSortedTagOrdinals(fullTagsList, tagsOrdinalInSortedOrder, tag, pbMagicPrefix); + } + return fullTagsList; + } + + private static void getSortedTagOrdinals(List> fullTagsList, + List tagsOrdinalInSortedOrder, Tag tag, boolean pbMagicPrefix) throws IOException { + if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { + int offset = tag.getTagOffset(); + int endOffset = offset + tag.getTagLength(); + if (pbMagicPrefix) { + offset += PBUFLEN; + } + while (offset < endOffset) { + Pair result = StreamUtils.readRawVarint32(tag.getBuffer(), offset); + tagsOrdinalInSortedOrder.add(result.getFirst()); + offset += result.getSecond(); + } + Collections.sort(tagsOrdinalInSortedOrder); + fullTagsList.add(cloneList(tagsOrdinalInSortedOrder)); + tagsOrdinalInSortedOrder.clear(); + } + } + + private static boolean compareTagsOrdinals(List> tagsInDeletes, + List> tags) { + boolean matchFound = false; + if (tagsInDeletes.size() != tags.size()) { + return matchFound; + } else { + for (List deleteTagOrdinals : tagsInDeletes) { + matchFound = false; + for (List tagOrdinals : tags) { + if (deleteTagOrdinals.containsAll(tagOrdinals)) { + matchFound = true; + break; + } + } + } + return matchFound; + } + } + + private static List cloneList(List tagsOrdinalList) { + List clone = new LinkedList(tagsOrdinalList); + return clone; + } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java new file mode 100644 index 0000000..3e3b098 --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java @@ -0,0 +1,1520 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +/** + * Tests visibility labels with deletes + */ +@Category(MediumTests.class) +public class TestVisibilityLabelsWithDeletes { + private static final String TOPSECRET = "TOPSECRET"; + private static final String PUBLIC = "PUBLIC"; + private static final String PRIVATE = "PRIVATE"; + private static final String CONFIDENTIAL = "CONFIDENTIAL"; + private static final String SECRET = "SECRET"; + public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final byte[] row1 = Bytes.toBytes("row1"); + private static final byte[] row2 = Bytes.toBytes("row2"); + private static final byte[] row3 = Bytes.toBytes("row3"); + private static final byte[] row4 = Bytes.toBytes("row4"); + private final static byte[] fam = Bytes.toBytes("info"); + private final static byte[] qual = Bytes.toBytes("qual"); + private final static byte[] value = Bytes.toBytes("value"); + public static Configuration conf; + + @Rule + public final TestName TEST_NAME = new TestName(); + public static User SUPERUSER; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // setup configuration + conf = TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); + conf.setBoolean("hbase.online.schema.update.enable", true); + conf.setInt("hfile.format.version", 3); + conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName()); + conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName()); + conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, + ScanLabelGenerator.class); + conf.set("hbase.superuser", "admin"); + TEST_UTIL.startMiniCluster(2); + SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); + + // Wait for the labels table to become available + TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000); + addLabels(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testVisibilityLabelsWithDeleteColumns() throws Throwable { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + TOPSECRET, + SECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(TOPSECRET + "&" + SECRET)); + d.deleteColumns(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteFamily() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET, CONFIDENTIAL + "|" + + TOPSECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row2); + d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL)); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteFamilyVersion() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + long[] ts = new long[] { 123l, 125l }; + final HTable table = createTableAndWriteDataWithLabels(tableName, ts, CONFIDENTIAL + "|" + + TOPSECRET, SECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL)); + d.deleteFamilyVersion(fam, 123l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteColumnExactVersion() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + long[] ts = new long[] { 123l, 125l }; + final HTable table = createTableAndWriteDataWithLabels(tableName, ts, CONFIDENTIAL + "|" + + TOPSECRET, SECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL)); + d.deleteColumn(fam, qual, 123l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteColumnsWithMultipleVersions() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteColumns(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + private HTable doPuts(TableName tableName) throws IOException, InterruptedIOException, + RetriesExhaustedWithDetailsException, InterruptedException { + HTable table; + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 125l, value); + put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 126l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + put = new Put(Bytes.toBytes("row2")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + TOPSECRET + + "&" + SECRET + ")")); + table.put(put); + return table; + } + + private HTable doPutsWithoutVisibility(TableName tableName) throws IOException, + InterruptedIOException, RetriesExhaustedWithDetailsException, InterruptedException { + HTable table; + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 125l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 126l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 127l, value); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + put = new Put(Bytes.toBytes("row2")); + put.add(fam, qual, 127l, value); + table.put(put); + return table; + } + + + @Test + public void testDeleteColumnWithSpecificTimeStampUsingMultipleVersionsUnMatchingVisExpression() + throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteColumn(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnWithLatestTimeStampUsingMultipleVersions() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnWithLatestTimeStampUsingMultipleVersionsAfterCompaction() + throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Put put = new Put(Bytes.toBytes("row3")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE)); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + TEST_UTIL.getHBaseAdmin().majorCompact(tableName.getNameAsString()); + // Sleep to ensure compaction happens. Need to do it in a better way + Thread.sleep(5000); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 3); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilyLatestTimeStampWithMulipleVersions() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilyWithoutCellVisibilityWithMulipleVersions() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPutsWithoutVisibility(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + // All cells wrt row1 should be deleted as we are not passing the Cell Visibility + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilyLatestTimeStampWithMulipleVersionsWithoutCellVisibilityInPuts() + throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPutsWithoutVisibility(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + @Test + public void testDeleteFamilySpecificTimeStampWithMulipleVersions() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteFamily(fam, 126l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testScanAfterCompaction() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteFamily(fam, 126l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Put put = new Put(Bytes.toBytes("row3")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE)); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + TEST_UTIL.getHBaseAdmin().compact(tableName.getNameAsString()); + Thread.sleep(5000); + // Sleep to ensure compaction happens. Need to do it in a better way + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 3); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilySpecificTimeStampWithMulipleVersionsDoneTwice() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteFamily(fam, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteFamily(fam, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 1); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + assertEquals(current.getTimestamp(), 127l); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDiffDeleteTypesForTheSameCellUsingMultipleVersions() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteColumns(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteColumn(fam, qual, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + @Test + public void testDeleteColumnLatestWithNoCellVisibility() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityExpressionWithNotEqualORCondition() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "|" + PRIVATE)); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteColumn(fam, qual, 124l); + d.setCellVisibility(new CellVisibility(PRIVATE )); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testORBasedVisibilityExpression () throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + put.setCellVisibility(new CellVisibility("(" + SECRET + "&" + TOPSECRET + ")|(" + CONFIDENTIAL + + "&" + PRIVATE + ")")); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteColumn(fam, qual, 123l); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + TOPSECRET + + "&" + SECRET + ")")); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 0); + }finally { + if (table != null) { + table.close(); + } + } + } + public static HTable createTableAndWriteDataWithLabels(TableName tableName, String... labelExps) + throws Exception { + HTable table = null; + try { + table = TEST_UTIL.createTable(tableName, fam); + int i = 1; + List puts = new ArrayList(); + for (String labelExp : labelExps) { + Put put = new Put(Bytes.toBytes("row" + i)); + put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value); + put.setCellVisibility(new CellVisibility(labelExp)); + puts.add(put); + table.put(put); + i++; + } + // table.put(puts); + } finally { + if (table != null) { + table.close(); + } + } + return table; + } + + public static HTable createTableAndWriteDataWithLabels(TableName tableName, long[] timestamp, + String... labelExps) throws Exception { + HTable table = null; + try { + table = TEST_UTIL.createTable(tableName, fam); + int i = 1; + List puts = new ArrayList(); + for (String labelExp : labelExps) { + Put put = new Put(Bytes.toBytes("row" + i)); + put.add(fam, qual, timestamp[i - 1], value); + put.setCellVisibility(new CellVisibility(labelExp)); + puts.add(put); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + i++; + } + } finally { + if (table != null) { + table.close(); + } + } + return table; + } + + public static void addLabels() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE }; + try { + VisibilityClient.addLabels(conf, labels); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(action); + } +}