diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index 1ffb12b..b3c5f83 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -404,6 +404,14 @@ public final class CellUtil { return cell.getTypeByte() == Type.DeleteFamilyVersion.getCode(); } + public static boolean isDeleteColumns(final Cell cell) { + return cell.getTypeByte() == Type.DeleteColumn.getCode(); + } + + public static boolean isDeleteColumn(final Cell cell) { + return cell.getTypeByte() == Type.Delete.getCode(); + } + /** * @param cell * @return Estimate of the cell size in bytes. diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java index 6c43c78..8842e16 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java @@ -27,4 +27,5 @@ public final class TagType { public static final byte ACL_TAG_TYPE = (byte) 1; public static final byte VISIBILITY_TAG_TYPE = (byte) 2; public static final byte LOG_REPLAY_TAG_TYPE = (byte) 3; + public static final byte VISBILITY_EXP_SERIALIZATION_TAG_TYPE = (byte)4; } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java index 3425a12..f726be1 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -1225,8 +1225,8 @@ public interface RegionObserver extends Coprocessor { Cell postMutationBeforeWAL(ObserverContext ctx, MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException; - /** - * Called after the ScanQueryMatcher creates ScanDeleteTracker. Implementing + + /* Called after the ScanQueryMatcher creates ScanDeleteTracker. Implementing * this hook would help in creating customised DeleteTracker and returning * the newly created DeleteTracker * diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java index 0c133f6..2b7e8d2 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java @@ -71,8 +71,8 @@ public class LabelExpander { // TODO : The code repeats from that in Visibility Controller.. Refactoring // may be needed - public List createVisibilityTags(String visibilityLabelsExp) throws IOException, - BadTsvLineException { + public List createVisibilityTags(String visibilityLabelsExp) + throws IOException, BadTsvLineException { ExpressionNode node = null; try { node = parser.parse(visibilityLabelsExp); @@ -83,6 +83,8 @@ public class LabelExpander { List tags = new ArrayList(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); + tags.add(new Tag(VisibilityUtils.VISBILITY_EXP_SERIALIZATION_TAG_TYPE, + VisibilityUtils.serializationArray)); if (node.isSingleNode()) { writeLabelOrdinalsToStream(node, dos); tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); @@ -92,7 +94,8 @@ public class LabelExpander { if (nlNode.getOperator() == Operator.OR) { for (ExpressionNode child : nlNode.getChildExps()) { writeLabelOrdinalsToStream(child, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, + baos.toByteArray())); baos.reset(); } } else { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java index 514628d..469d451 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java @@ -45,7 +45,7 @@ public interface DeleteTracker { /** * Check if the specified cell buffer has been deleted by a previously * seen delete. - * @param cell - current cell to check if deleted by a previously deleted cell + * @param cell - current cell to check if deleted by a previously seen delete * @return deleteResult The result tells whether the KeyValue is deleted and why */ DeleteResult isDeleted(Cell cell); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 0c4f7ee..a669ec9 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -2063,8 +2063,8 @@ public class HRegion implements HeapSize { // , Writable{ get.setMaxVersions(count); get.addColumn(family, qual); if (coprocessorHost != null) { - if (!coprocessorHost.prePrepareTimeStampForDeleteVersion(mutation, cell, byteNow, - get)) { + if (!coprocessorHost.prePrepareTimeStampForDeleteVersion(mutation, cell, + byteNow, get)) { updateDeleteLatestVersionTimeStamp(kv, get, count, byteNow); } } else { @@ -4752,7 +4752,7 @@ public class HRegion implements HeapSize { // , Writable{ * @param withCoprocessor invoke coprocessor or not. We don't want to * always invoke cp for this private method. */ - private List get(Get get, boolean withCoprocessor) + public List get(Get get, boolean withCoprocessor) throws IOException { List results = new ArrayList(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java index 6a767e8..668e3db 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java @@ -45,14 +45,14 @@ import org.apache.hadoop.hbase.util.Bytes; @InterfaceAudience.Private public class ScanDeleteTracker implements DeleteTracker { - private boolean hasFamilyStamp = false; - private long familyStamp = 0L; - private SortedSet familyVersionStamps = new TreeSet(); - private byte [] deleteBuffer = null; - private int deleteOffset = 0; - private int deleteLength = 0; - private byte deleteType = 0; - private long deleteTimestamp = 0L; + protected boolean hasFamilyStamp = false; + protected long familyStamp = 0L; + protected SortedSet familyVersionStamps = new TreeSet(); + protected byte [] deleteBuffer = null; + protected int deleteOffset = 0; + protected int deleteLength = 0; + protected byte deleteType = 0; + protected long deleteTimestamp = 0L; /** * Constructor for ScanDeleteTracker @@ -65,7 +65,7 @@ public class ScanDeleteTracker implements DeleteTracker { * Add the specified KeyValue to the list of deletes to check against for * this row operation. *

- * This is called when a Delete is encountered in a StoreFile. + * This is called when a Delete is encountered. * @param cell - the delete cell */ @Override diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index a152fcc..cf2b9b2 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -30,6 +30,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -75,6 +76,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionObserver; import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterBase; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.util.StreamUtils; @@ -93,6 +95,7 @@ import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.Visibil import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.DeleteTracker; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -149,7 +152,6 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb private boolean acOn = false; private Configuration conf; private volatile boolean initialized = false; - /** Mapping of scanner instances to the user who created them */ private Map scannerOwners = new MapMaker().weakKeys().makeMap(); @@ -688,10 +690,8 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb new OperationStatus(SANITY_CHECK_FAILURE, de.getMessage())); continue; } - if (m instanceof Put) { - Put p = (Put) m; boolean sanityFailure = false; - for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) { + for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, "Mutation contains cell with reserved type tag")); @@ -717,7 +717,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (visibilityTags != null) { labelCache.put(labelsExp, visibilityTags); List updatedCells = new ArrayList(); - for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) { + for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { Cell cell = cellScanner.current(); List tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); @@ -730,22 +730,68 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb cell.getValueLength(), tags); updatedCells.add(updatedCell); } - p.getFamilyCellMap().clear(); - // Clear and add new Cells to the Mutation. - for (Cell cell : updatedCells) { + m.getFamilyCellMap().clear(); + // Clear and add new Cells to the Mutation. + for (Cell cell : updatedCells) { + if (m instanceof Put) { + Put p = (Put) m; p.add(cell); + } else if (m instanceof Delete) { + // Should we fail any delete without CellVisibility? + Delete d = (Delete) m; + d.addDeleteMarker(cell); } } } } - } else if (cellVisibility != null) { - // CellVisibility in a Delete is not legal! Fail the operation - miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, - "CellVisibility cannot be set on Delete mutation")); } } } + + @Override + public void prePrepareTimeStampForDeleteVersion( + ObserverContext ctx, Mutation delete, Cell cell, + byte[] byteNow, Get get) throws IOException { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + CellVisibility cellVisibility = null; + try { + cellVisibility = delete.getCellVisibility(); + } catch (DeserializationException de) { + throw new RuntimeException("Invalid cell visibility specified " + delete); + } + for (CellScanner cellScanner = delete.cellScanner(); cellScanner.advance();) { + if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { + throw new RuntimeException("Reserved visibility type used " + cellVisibility.toString()); + } + } + List visibilityTags = new ArrayList(); + if (cellVisibility != null) { + String labelsExp = cellVisibility.getExpression(); + try { + visibilityTags = createVisibilityTags(labelsExp); + } catch (ParseException e) { + throw new RuntimeException("Invalid cell visibility expression " + labelsExp); + } catch (InvalidLabelException e) { + throw new RuntimeException("Invalid cell visibility specified " + labelsExp); + } + } + get.setFilter(new DeleteVersionVisibilityExpressionFilter(visibilityTags)); + List result = ctx.getEnvironment().getRegion().get(get, false); + if (result.size() < get.getMaxVersions()) { + // Nothing to delete + kv.updateLatestStamp(Bytes.toBytes(Long.MIN_VALUE)); + return; + } + if (result.size() > get.getMaxVersions()) { + throw new RuntimeException("Unexpected size: " + result.size()); + } + KeyValue getkv = KeyValueUtil.ensureKeyValue(result.get(get.getMaxVersions() - 1)); + Bytes.putBytes(kv.getBuffer(), kv.getTimestampOffset(), getkv.getBuffer(), + getkv.getTimestampOffset(), Bytes.SIZEOF_LONG); + ctx.bypass(); + } + @Override public void postBatchMutate(ObserverContext c, MiniBatchOperationInProgress miniBatchOp) throws IOException { @@ -844,36 +890,54 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb return true; } - private List createVisibilityTags(String visibilityLabelsExp) throws IOException, - ParseException, InvalidLabelException { + private List createVisibilityTags(String visibilityLabelsExp) + throws IOException, ParseException, InvalidLabelException { ExpressionNode node = null; node = this.expressionParser.parse(visibilityLabelsExp); node = this.expressionExpander.expand(node); List tags = new ArrayList(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); + List labelOrdinals = new ArrayList(); + // Add new tag type + tags.add(new Tag(VisibilityUtils.VISBILITY_EXP_SERIALIZATION_TAG_TYPE, + VisibilityUtils.serializationArray)); if (node.isSingleNode()) { - writeLabelOrdinalsToStream(node, dos); + getLabelOrdinals(node, labelOrdinals); + writeLabelOrdinalsToStream(labelOrdinals, dos); tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); baos.reset(); } else { NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node; if (nlNode.getOperator() == Operator.OR) { for (ExpressionNode child : nlNode.getChildExps()) { - writeLabelOrdinalsToStream(child, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + getLabelOrdinals(child, labelOrdinals); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, + baos.toByteArray())); baos.reset(); + labelOrdinals.clear(); } } else { - writeLabelOrdinalsToStream(nlNode, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + getLabelOrdinals(nlNode, labelOrdinals); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, + baos.toByteArray())); baos.reset(); } } return tags; } - private void writeLabelOrdinalsToStream(ExpressionNode node, DataOutputStream dos) + private void writeLabelOrdinalsToStream(List labelOrdinals, DataOutputStream dos) + throws IOException { + Collections.sort(labelOrdinals); + for (Integer labelOrdinal : labelOrdinals) { + StreamUtils.writeRawVInt32(dos, labelOrdinal); + } + } + + private void getLabelOrdinals(ExpressionNode node, List labelOrdinals) throws IOException, InvalidLabelException { if (node.isSingleNode()) { String identifier = null; @@ -896,11 +960,11 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (labelOrdinal == 0) { throw new InvalidLabelException("Invalid visibility label " + identifier); } - StreamUtils.writeRawVInt32(dos, labelOrdinal); + labelOrdinals.add(labelOrdinal); } else { List childExps = ((NonLeafExpressionNode) node).getChildExps(); for (ExpressionNode child : childExps) { - writeLabelOrdinalsToStream(child, dos); + getLabelOrdinals(child, labelOrdinals); } } } @@ -942,6 +1006,17 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } @Override + public DeleteTracker postInstantiateDeleteTracker( + ObserverContext ctx, DeleteTracker delTracker) + throws IOException { + HRegion region = ctx.getEnvironment().getRegion(); + TableName table = region.getRegionInfo().getTable(); + if (table.isSystemTable() || table.equals(LABELS_TABLE_NAME)) { + return delTracker; + } + return new VisibilityScanDeleteTracker(); + } + @Override public RegionScanner postScannerOpen(final ObserverContext c, final Scan scan, final RegionScanner s) throws IOException { User user = getActiveUser(); @@ -1409,4 +1484,22 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } } } + + static class DeleteVersionVisibilityExpressionFilter extends FilterBase { + private List visibilityTags; + + public DeleteVersionVisibilityExpressionFilter(List visibilityTags) { + this.visibilityTags = visibilityTags; + } + + @Override + public ReturnCode filterKeyValue(Cell kv) throws IOException { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(kv, visibilityTags, true); + if (matchFound) { + return ReturnCode.INCLUDE; + } else { + return ReturnCode.SKIP; + } + } + } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java new file mode 100644 index 0000000..e74695f --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java @@ -0,0 +1,272 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Similar to ScanDeletTracker but tracks the visibility expression also before + * deciding if a Cell can be considered deleted + */ +public class VisibilityScanDeleteTracker extends ScanDeleteTracker { + + // Its better to track the visibility tags in delete based on each type. Create individual + // data structures for tracking each of them. This would ensure that there is no tracking based + // on time and also would handle all cases where deletefamily or deletecolumns is specified with + // Latest_timestamp. In such cases the ts in the delete marker and the masking + // put will not be same. So going with individual data structures for different delete + // type would solve this problem and also ensure that the combination of different type + // of deletes with diff ts would also work fine + private List visibilityTagsDeleteFamily; + private List visibilityTagsDeleteFamilyVersion; + private List visibilityTagsDeleteColumns; + private List visiblityTagsDeleteColumn; + + public VisibilityScanDeleteTracker() { + super(); + } + + @Override + public void add(Cell delCell) { + //Cannot call super.add because need to find if the delete needs to be considered + long timestamp = delCell.getTimestamp(); + int qualifierOffset = delCell.getQualifierOffset(); + int qualifierLength = delCell.getQualifierLength(); + byte type = delCell.getTypeByte(); + if (!hasFamilyStamp || timestamp > familyStamp) { + if (type == KeyValue.Type.DeleteFamily.getCode()) { + hasFamilyStamp = true; + familyStamp = timestamp; + extractDeleteTags(delCell, KeyValue.Type.DeleteFamily); + return; + } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) { + familyVersionStamps.add(timestamp); + extractDeleteTags(delCell, KeyValue.Type.DeleteFamilyVersion); + return; + } + + if (deleteBuffer != null && type < deleteType) { + // same column, so ignore less specific delete + if (Bytes.equals(deleteBuffer, deleteOffset, deleteLength, + delCell.getQualifierArray(), qualifierOffset, qualifierLength)){ + return; + } + } + // new column, or more general delete type + deleteBuffer = delCell.getQualifierArray(); + deleteOffset = qualifierOffset; + deleteLength = qualifierLength; + deleteType = type; + deleteTimestamp = timestamp; + extractDeleteTags(delCell, KeyValue.Type.codeToType(type)); + } + } + + private void extractDeleteTags(Cell delCell, Type type) { + // If tag is present in the delete + if (delCell.getTagsLength() > 0) { + switch (type) { + case DeleteFamily: + visibilityTagsDeleteFamily = new ArrayList(); + VisibilityUtils.getVisibilityTags(delCell, visibilityTagsDeleteFamily); + if(visibilityTagsDeleteFamily.isEmpty()) { + visibilityTagsDeleteFamily = null; + } + break; + case DeleteFamilyVersion: + visibilityTagsDeleteFamilyVersion = new ArrayList(); + VisibilityUtils.getVisibilityTags(delCell, visibilityTagsDeleteFamilyVersion); + if(visibilityTagsDeleteFamilyVersion.isEmpty()) { + visibilityTagsDeleteFamilyVersion = null; + } + break; + case DeleteColumn: + visibilityTagsDeleteColumns = new ArrayList(); + VisibilityUtils.getVisibilityTags(delCell, visibilityTagsDeleteColumns); + if(visibilityTagsDeleteColumns.isEmpty()) { + visibilityTagsDeleteColumns = null; + } + break; + case Delete: + visiblityTagsDeleteColumn = new ArrayList(); + VisibilityUtils.getVisibilityTags(delCell, visiblityTagsDeleteColumn); + if(visiblityTagsDeleteColumn.isEmpty()) { + visiblityTagsDeleteColumn = null; + } + break; + default: + throw new IllegalArgumentException("Invalid delete type"); + } + } else { + switch (type) { + case DeleteFamily: + visibilityTagsDeleteFamily = null; + break; + case DeleteFamilyVersion: + visibilityTagsDeleteFamilyVersion = null; + break; + case DeleteColumn: + visibilityTagsDeleteColumns = null; + break; + case Delete: + visiblityTagsDeleteColumn = null; + break; + default: + throw new IllegalArgumentException("Invalid delete type"); + } + } + } + + @Override + public DeleteResult isDeleted(Cell cell) { + long timestamp = cell.getTimestamp(); + int qualifierOffset = cell.getQualifierOffset(); + int qualifierLength = cell.getQualifierLength(); + if (hasFamilyStamp && timestamp <= familyStamp) { + if (visibilityTagsDeleteFamily != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsDeleteFamily, false); + if (matchFound) { + // Should we return FAMILY_VERSION_DELETED or FAMILY_DELETED here + return DeleteResult.FAMILY_VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + + } else { + List tags = new ArrayList(); + VisibilityUtils.getVisibilityTags(cell, tags); + if (tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.FAMILY_DELETED; + } + } + } + + if (familyVersionStamps.contains(Long.valueOf(timestamp))) { + if (visibilityTagsDeleteFamilyVersion != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsDeleteFamilyVersion, false); + if (matchFound) { + return DeleteResult.FAMILY_VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + + } else { + List tags = new ArrayList(); + VisibilityUtils.getVisibilityTags(cell, tags); + if (tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.FAMILY_VERSION_DELETED; + } + } + } + + if (deleteBuffer != null) { + int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, cell.getQualifierArray(), + qualifierOffset, qualifierLength); + + if (ret == 0) { + if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { + if (visibilityTagsDeleteColumns != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsDeleteColumns, false); + if (matchFound) { + // Should we return COLUMN_DELETED or VERSION_DELETED + return DeleteResult.VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + + } else { + List tags = new ArrayList(); + VisibilityUtils.getVisibilityTags(cell, tags); + if (tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.COLUMN_DELETED; + } + } + } + // Delete (aka DeleteVersion) + // If the timestamp is the same, keep this one + if (timestamp == deleteTimestamp) { + if (visiblityTagsDeleteColumn != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visiblityTagsDeleteColumn, false); + if (matchFound) { + return DeleteResult.VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + } else { + List tags = new ArrayList(); + VisibilityUtils.getVisibilityTags(cell, tags); + if (tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.VERSION_DELETED; + } + } + } + // use assert or not? + assert timestamp < deleteTimestamp; + + // different timestamp, let's clear the buffer. + deleteBuffer = null; + visiblityTagsDeleteColumn = null; + } else if (ret < 0) { + // Next column case. + deleteBuffer = null; + visibilityTagsDeleteColumns = null; + visiblityTagsDeleteColumn = null; + } else { + throw new IllegalStateException("isDelete failed: deleteBuffer=" + + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier=" + + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength) + + ", timestamp=" + timestamp + ", comparison result: " + ret); + } + } + return DeleteResult.NOT_DELETED; + } + + @Override + public void reset() { + super.reset(); + visibilityTagsDeleteColumns = null; + visibilityTagsDeleteFamily = null; + visibilityTagsDeleteFamilyVersion = null; + visiblityTagsDeleteColumn = null; + } +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java index 0c7764f..16bd629 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java @@ -19,25 +19,32 @@ package org.apache.hadoop.hbase.security.visibility; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import com.google.protobuf.HBaseZeroCopyByteString; - import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.util.ReflectionUtils; +import com.google.protobuf.HBaseZeroCopyByteString; import com.google.protobuf.InvalidProtocolBufferException; /** @@ -46,9 +53,14 @@ import com.google.protobuf.InvalidProtocolBufferException; @InterfaceAudience.Private public class VisibilityUtils { - public static final String VISIBILITY_LABEL_GENERATOR_CLASS = + public static final String VISIBILITY_LABEL_GENERATOR_CLASS = "hbase.regionserver.scan.visibility.label.generator.class"; public static final byte VISIBILITY_TAG_TYPE = TagType.VISIBILITY_TAG_TYPE; + public static final byte VISBILITY_EXP_SERIALIZATION_TAG_TYPE = + TagType.VISBILITY_EXP_SERIALIZATION_TAG_TYPE; + public static final int VISIBILITY_SERIALIZATION_VERSION = 1; + public static final byte[] serializationArray = Bytes + .toBytes(VISIBILITY_SERIALIZATION_VERSION); public static final String SYSTEM_LABEL = "system"; private static final String COMMA = ","; @@ -156,4 +168,168 @@ public class VisibilityUtils { } return slgs; } + + /** + * Get the list of visibility tags in the given cell + * @param cell - the cell + * @param tags - the tags array that will be populated if + * visibility tags are present + * @return + */ + public static boolean getVisibilityTags(Cell cell, List tags) { + boolean sortedOrder = false; + byte[] tagBytes = CellUtil.getTagArray(cell); + Iterator tagsIterator = CellUtil.tagsIterator(tagBytes, 0, tagBytes.length); + while (tagsIterator.hasNext()) { + Tag tag = tagsIterator.next(); + if(tag.getType() == VisibilityUtils.VISBILITY_EXP_SERIALIZATION_TAG_TYPE) { + int serializationVersion = Bytes.toInt(tag.getValue()); + if (serializationVersion == VisibilityUtils.VISIBILITY_SERIALIZATION_VERSION) { + sortedOrder = true; + continue; + } + } + if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { + tags.add(tag); + } + } + return sortedOrder; + } + + /** + * Checks for the matching visibility labels in the delete mutation and + * the cell in consideration + * @param cell - the cell + * @param visibilityTagsInDeleteCell - that list of tags in the delete mutation + * (the specified Cell Visibility) + * @param newTagPresentInDelete - If the delete mutation has tag in it or not + * @return + */ + public static boolean checkForMatchingVisibilityTags(Cell cell, + List visibilityTagsInDeleteCell, boolean newTagPresentInDelete) { + List tags = new ArrayList(); + boolean sortedTags = getVisibilityTags(cell, tags); + if (tags.size() == 0) { + // Early out if there are no tags in the cell + return false; + } + if (sortedTags) { + return checkForMatchingVisibilityTagsWithSortedOrder(cell.getTagsLength(), + visibilityTagsInDeleteCell, tags, newTagPresentInDelete); + } else { + try { + return checkForMatchingVisibilityTagsWithOutSortedOrder(cell, visibilityTagsInDeleteCell); + } catch (IOException e) { + // Should not happen + throw new RuntimeException("Exception while sorting the tags from the cell", e); + } + } + } + + private static boolean checkForMatchingVisibilityTagsWithOutSortedOrder(Cell cell, + List visibilityTagsInDeleteCell) throws IOException { + List> sortedDeleteTags = sortTagsBasedOnOrdinal( + visibilityTagsInDeleteCell); + List> sortedTags = sortedTagsBasedOnOrdinal(cell); + return compareTagsOrdinals(sortedDeleteTags, sortedTags); + } + + private static boolean checkForMatchingVisibilityTagsWithSortedOrder(short tagsLength, + List visibilityTagsInDeleteCell, List tags, boolean newTagPresentInDelete) { + boolean matchFound = false; + if (tagsLength > 0) { + // the tags in visibility delete cell will also include the new type of tag + if (newTagPresentInDelete) { + if ((visibilityTagsInDeleteCell.size() - 1) != tags.size()) { + // If the size does not match. Definitely we are not comparing the + // equal + // tags. + // Return false in that case. + return matchFound; + } + } else { + if ((visibilityTagsInDeleteCell.size()) != tags.size()) { + // If the size does not match. Definitely we are not comparing the + // equal + // tags. + // Return false in that case. + return matchFound; + } + } + for (Tag tag : visibilityTagsInDeleteCell) { + matchFound = false; + for (Tag givenTag : tags) { + if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(), + givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) { + matchFound = true; + break; + } + } + } + } + return matchFound; + } + + private static List> sortedTagsBasedOnOrdinal(Cell cell) + throws IOException { + Iterator tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), + cell.getTagsLength()); + List> fullTagsList = new ArrayList>(); + List tagsOrdinalInSortedOrder = new ArrayList(); + while (tagsItr.hasNext()) { + Tag tag = tagsItr.next(); + getSortedTagOrdinals(fullTagsList, tagsOrdinalInSortedOrder, tag); + } + return fullTagsList; + } + + private static List> sortTagsBasedOnOrdinal(List tags) + throws IOException { + List> fullTagsList = new ArrayList>(); + List tagsOrdinalInSortedOrder = new ArrayList(); + for (Tag tag : tags) { + getSortedTagOrdinals(fullTagsList, tagsOrdinalInSortedOrder, tag); + } + return fullTagsList; + } + + private static void getSortedTagOrdinals(List> fullTagsList, + List tagsOrdinalInSortedOrder, Tag tag) throws IOException { + if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { + int offset = tag.getTagOffset(); + int endOffset = offset + tag.getTagLength(); + while (offset < endOffset) { + Pair result = StreamUtils.readRawVarint32(tag.getBuffer(), offset); + tagsOrdinalInSortedOrder.add(result.getFirst()); + offset += result.getSecond(); + } + Collections.sort(tagsOrdinalInSortedOrder); + fullTagsList.add(cloneList(tagsOrdinalInSortedOrder)); + tagsOrdinalInSortedOrder.clear(); + } + } + + private static boolean compareTagsOrdinals(List> tagsInDeletes, + List> tags) { + boolean matchFound = false; + if (tagsInDeletes.size() != tags.size()) { + return matchFound; + } else { + for (List deleteTagOrdinals : tagsInDeletes) { + matchFound = false; + for (List tagOrdinals : tags) { + if (deleteTagOrdinals.containsAll(tagOrdinals)) { + matchFound = true; + break; + } + } + } + return matchFound; + } + } + + private static List cloneList(List tagsOrdinalList) { + List clone = new LinkedList(tagsOrdinalList); + return clone; + } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java new file mode 100644 index 0000000..d5d778a --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java @@ -0,0 +1,1858 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +/** + * Tests visibility labels with deletes + */ +@Category(MediumTests.class) +public class TestVisibilityLabelsWithDeletes { + private static final String TOPSECRET = "TOPSECRET"; + private static final String PUBLIC = "PUBLIC"; + private static final String PRIVATE = "PRIVATE"; + private static final String CONFIDENTIAL = "CONFIDENTIAL"; + private static final String SECRET = "SECRET"; + public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final byte[] row1 = Bytes.toBytes("row1"); + private static final byte[] row2 = Bytes.toBytes("row2"); + private final static byte[] fam = Bytes.toBytes("info"); + private final static byte[] qual = Bytes.toBytes("qual"); + private final static byte[] qual1 = Bytes.toBytes("qual1"); + private final static byte[] qual2 = Bytes.toBytes("qual2"); + private final static byte[] value = Bytes.toBytes("value"); + public static Configuration conf; + + @Rule + public final TestName TEST_NAME = new TestName(); + public static User SUPERUSER; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // setup configuration + conf = TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); + conf.setBoolean("hbase.online.schema.update.enable", true); + conf.setInt("hfile.format.version", 3); + conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName()); + conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName()); + conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, + ScanLabelGenerator.class); + conf.set("hbase.superuser", "admin"); + TEST_UTIL.startMiniCluster(2); + SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); + + // Wait for the labels table to become available + TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000); + addLabels(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testVisibilityLabelsWithDeleteColumns() throws Throwable { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + TOPSECRET, + SECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(TOPSECRET + "&" + SECRET)); + d.deleteColumns(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteFamily() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET, CONFIDENTIAL + "|" + + TOPSECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row2); + d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL)); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteFamilyVersion() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + long[] ts = new long[] { 123l, 125l }; + final HTable table = createTableAndWriteDataWithLabels(tableName, ts, CONFIDENTIAL + "|" + + TOPSECRET, SECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL)); + d.deleteFamilyVersion(fam, 123l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteColumnExactVersion() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + long[] ts = new long[] { 123l, 125l }; + final HTable table = createTableAndWriteDataWithLabels(tableName, ts, CONFIDENTIAL + "|" + + TOPSECRET, SECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL)); + d.deleteColumn(fam, qual, 123l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteColumnsWithMultipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteColumns(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + private HTable doPuts(TableName tableName) throws IOException, InterruptedIOException, + RetriesExhaustedWithDetailsException, InterruptedException { + HTable table; + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 125l, value); + put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 126l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + put = new Put(Bytes.toBytes("row2")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + TOPSECRET + + "&" + SECRET + ")")); + table.put(put); + return table; + } + + private HTable doPutsWithDiffCols(TableName tableName) throws IOException, + InterruptedIOException, RetriesExhaustedWithDetailsException, InterruptedException { + HTable table; + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 125l, value); + put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual1, 126l, value); + put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual2, 127l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + return table; + } + + private HTable doPutsWithoutVisibility(TableName tableName) throws IOException, + InterruptedIOException, RetriesExhaustedWithDetailsException, InterruptedException { + HTable table; + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 125l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 126l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 127l, value); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + put = new Put(Bytes.toBytes("row2")); + put.add(fam, qual, 127l, value); + table.put(put); + return table; + } + + + @Test + public void testDeleteColumnWithSpecificTimeStampUsingMultipleVersionsUnMatchingVisExpression() + throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteColumn(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnWithLatestTimeStampUsingMultipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + + @Test (timeout=1800000) + public void testDeleteColumnWithLatestTimeStampWhenNoVersionMatches() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 128l, value); + put.setCellVisibility(new CellVisibility(TOPSECRET)); + table.put(put); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET )); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 128l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 129l, value); + put.setCellVisibility(new CellVisibility(SECRET)); + table.put(put); + table.flushCommits(); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 129l); + } finally { + if (table != null) { + table.close(); + } + } + } + @Test + public void testDeleteColumnWithLatestTimeStampUsingMultipleVersionsAfterCompaction() + throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Put put = new Put(Bytes.toBytes("row3")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE)); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + TEST_UTIL.getHBaseAdmin().majorCompact(tableName.getNameAsString()); + // Sleep to ensure compaction happens. Need to do it in a better way + Thread.sleep(5000); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 3); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilyLatestTimeStampWithMulipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnswithMultipleColumnsWithMultipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPutsWithDiffCols(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumns(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(), + current.getQualifierLength(), qual1, 0, qual1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(), + current.getQualifierLength(), qual2, 0, qual2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilyWithoutCellVisibilityWithMulipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPutsWithoutVisibility(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + // All cells wrt row1 should be deleted as we are not passing the Cell Visibility + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilyLatestTimeStampWithMulipleVersionsWithoutCellVisibilityInPuts() + throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPutsWithoutVisibility(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + @Test + public void testDeleteFamilySpecificTimeStampWithMulipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteFamily(fam, 126l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(6); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testScanAfterCompaction() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteFamily(fam, 126l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Put put = new Put(Bytes.toBytes("row3")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE)); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + TEST_UTIL.getHBaseAdmin().compact(tableName.getNameAsString()); + Thread.sleep(5000); + // Sleep to ensure compaction happens. Need to do it in a better way + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 3); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilySpecificTimeStampWithMulipleVersionsDoneTwice() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteFamily(fam, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteFamily(fam, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + assertEquals(current.getTimestamp(), 127l); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnSpecificTimeStampWithMulipleVersionsDoneTwice() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteColumn(fam, qual, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + assertEquals(current.getTimestamp(), 127l); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnSpecificTimeStampWithMulipleVersionsDoneTwice1() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")" + + "|(" + TOPSECRET + "&" + SECRET + ")")); + d.deleteColumn(fam, qual, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + assertEquals(current.getTimestamp(), 127l); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnAndDeleteFamilylSpecificTimeStampWithMulipleVersion() + throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteFamily(fam, 124l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + assertEquals(current.getTimestamp(), 127l); + } finally { + if (table != null) { + table.close(); + } + } + } + + private void setAuths() throws IOException, InterruptedException { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + } + + @Test + public void testDiffDeleteTypesForTheSameCellUsingMultipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteColumns(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteColumn(fam, qual, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + @Test + public void testDeleteColumnLatestWithNoCellVisibility() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityExpressionWithNotEqualORCondition() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "|" + PRIVATE)); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteColumn(fam, qual, 124l); + d.setCellVisibility(new CellVisibility(PRIVATE )); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + } finally { + if (table != null) { + table.close(); + } + } + } + + public static HTable createTableAndWriteDataWithLabels(TableName tableName, String... labelExps) + throws Exception { + HTable table = null; + try { + table = TEST_UTIL.createTable(tableName, fam); + int i = 1; + List puts = new ArrayList(); + for (String labelExp : labelExps) { + Put put = new Put(Bytes.toBytes("row" + i)); + put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value); + put.setCellVisibility(new CellVisibility(labelExp)); + puts.add(put); + table.put(put); + i++; + } + // table.put(puts); + } finally { + if (table != null) { + table.close(); + } + } + return table; + } + + public static HTable createTableAndWriteDataWithLabels(TableName tableName, long[] timestamp, + String... labelExps) throws Exception { + HTable table = null; + try { + table = TEST_UTIL.createTable(tableName, fam); + int i = 1; + List puts = new ArrayList(); + for (String labelExp : labelExps) { + Put put = new Put(Bytes.toBytes("row" + i)); + put.add(fam, qual, timestamp[i - 1], value); + put.setCellVisibility(new CellVisibility(labelExp)); + puts.add(put); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + i++; + } + } finally { + if (table != null) { + table.close(); + } + } + return table; + } + + public static void addLabels() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE }; + try { + VisibilityClient.addLabels(conf, labels); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(action); + } +}