diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java index 24ce293..f89fe17 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java @@ -265,6 +265,32 @@ public class Delete extends Mutation implements Comparable { } /** + * Add the specified KeyValue to this Delete operation. Operation assumes that + * the passed Cell is immutable and its backing array will not be modified + * for the duration of this Delete. Internal usage only. + * @param kv individual Cell + * @return this + * @throws java.io.IOException e + */ + public Delete delete(Cell kv) throws IOException { + byte[] family = CellUtil.cloneFamily(kv); + List list = getCellList(family); + // Checking that the row of the kv is the same as the put + if (!KeyValue.isDelete(kv.getTypeByte())) { + throw new WrongRowIOException("The cell " + kv.toString() + "is not of Delete type"); + } + int res = Bytes.compareTo(this.row, 0, row.length, kv.getRowArray(), kv.getRowOffset(), + kv.getRowLength()); + if (res != 0) { + throw new WrongRowIOException("The row in " + kv.toString() + + " doesn't match the original one " + Bytes.toStringBinary(this.row)); + } + list.add(kv); + familyMap.put(family, list); + return this; + } + + /** * Delete the latest version of the specified column. * This is an expensive call in that on the server-side, it first does a * get to find the latest versions timestamp. Then it adds a delete using diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index 1ffb12b..b3c5f83 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -404,6 +404,14 @@ public final class CellUtil { return cell.getTypeByte() == Type.DeleteFamilyVersion.getCode(); } + public static boolean isDeleteColumns(final Cell cell) { + return cell.getTypeByte() == Type.DeleteColumn.getCode(); + } + + public static boolean isDeleteColumn(final Cell cell) { + return cell.getTypeByte() == Type.Delete.getCode(); + } + /** * @param cell * @return Estimate of the cell size in bytes. diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java index 6c43c78..493fa00 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java @@ -27,4 +27,5 @@ public final class TagType { public static final byte ACL_TAG_TYPE = (byte) 1; public static final byte VISIBILITY_TAG_TYPE = (byte) 2; public static final byte LOG_REPLAY_TAG_TYPE = (byte) 3; + public static final byte DELETE_VISIBILITY_TYPE_FOUND = (byte)4; } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java index 96cc3bd..ef486ee 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -1111,8 +1111,8 @@ public interface RegionObserver extends Coprocessor { Cell postMutationBeforeWAL(ObserverContext ctx, MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException; - /** - * Called after the ScanQueryMatcher creates ScanDeleteTracker. Implementing + + /* Called after the ScanQueryMatcher creates ScanDeleteTracker. Implementing * this hook would help in creating customised DeleteTracker and returning * the newly created DeleteTracker * diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java index 0c133f6..eac42fd 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java @@ -31,6 +31,7 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.Tag; @@ -71,8 +72,8 @@ public class LabelExpander { // TODO : The code repeats from that in Visibility Controller.. Refactoring // may be needed - public List createVisibilityTags(String visibilityLabelsExp) throws IOException, - BadTsvLineException { + public List createVisibilityTags(String visibilityLabelsExp, boolean isDelete) + throws IOException, BadTsvLineException { ExpressionNode node = null; try { node = parser.parse(visibilityLabelsExp); @@ -83,6 +84,9 @@ public class LabelExpander { List tags = new ArrayList(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); + if (!isDelete) { + tags.add(new Tag(VisibilityUtils.DELETE_VISIBILITY_TYPE_FOUND, HConstants.EMPTY_BYTE_ARRAY)); + } if (node.isSingleNode()) { writeLabelOrdinalsToStream(node, dos); tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); @@ -92,7 +96,8 @@ public class LabelExpander { if (nlNode.getOperator() == Operator.OR) { for (ExpressionNode child : nlNode.getChildExps()) { writeLabelOrdinalsToStream(child, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, + baos.toByteArray())); baos.reset(); } } else { @@ -201,7 +206,7 @@ public class LabelExpander { KeyValue kv = null; if (cellVisibilityExpr != null) { // Apply the expansion and parsing here - List visibilityTags = createVisibilityTags(cellVisibilityExpr); + List visibilityTags = createVisibilityTags(cellVisibilityExpr, true); kv = new KeyValue(lineBytes, rowKeyOffset, rowKeyLength, family, familyOffset, familyLength, qualifier, qualifierOffset, qualifierLength, ts, KeyValue.Type.Put, lineBytes, columnOffset, columnLength, visibilityTags); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java index 514628d..469d451 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java @@ -45,7 +45,7 @@ public interface DeleteTracker { /** * Check if the specified cell buffer has been deleted by a previously * seen delete. - * @param cell - current cell to check if deleted by a previously deleted cell + * @param cell - current cell to check if deleted by a previously seen delete * @return deleteResult The result tells whether the KeyValue is deleted and why */ DeleteResult isDeleted(Cell cell); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java index 6a767e8..668e3db 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanDeleteTracker.java @@ -45,14 +45,14 @@ import org.apache.hadoop.hbase.util.Bytes; @InterfaceAudience.Private public class ScanDeleteTracker implements DeleteTracker { - private boolean hasFamilyStamp = false; - private long familyStamp = 0L; - private SortedSet familyVersionStamps = new TreeSet(); - private byte [] deleteBuffer = null; - private int deleteOffset = 0; - private int deleteLength = 0; - private byte deleteType = 0; - private long deleteTimestamp = 0L; + protected boolean hasFamilyStamp = false; + protected long familyStamp = 0L; + protected SortedSet familyVersionStamps = new TreeSet(); + protected byte [] deleteBuffer = null; + protected int deleteOffset = 0; + protected int deleteLength = 0; + protected byte deleteType = 0; + protected long deleteTimestamp = 0L; /** * Constructor for ScanDeleteTracker @@ -65,7 +65,7 @@ public class ScanDeleteTracker implements DeleteTracker { * Add the specified KeyValue to the list of deletes to check against for * this row operation. *

- * This is called when a Delete is encountered in a StoreFile. + * This is called when a Delete is encountered. * @param cell - the delete cell */ @Override diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index a344211..27462a4 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -30,10 +30,12 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -57,6 +59,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; @@ -93,12 +96,14 @@ import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.Visibil import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.DeleteTracker; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; import org.apache.hadoop.hbase.regionserver.OperationStatus; import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessControlLists; @@ -666,10 +671,8 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb new OperationStatus(SANITY_CHECK_FAILURE, de.getMessage())); continue; } - if (m instanceof Put) { - Put p = (Put) m; boolean sanityFailure = false; - for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) { + for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, "Mutation contains cell with reserved type tag")); @@ -683,7 +686,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb List visibilityTags = labelCache.get(labelsExp); if (visibilityTags == null) { try { - visibilityTags = createVisibilityTags(labelsExp); + visibilityTags = createVisibilityTags(labelsExp, m instanceof Delete); } catch (ParseException e) { miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, e.getMessage())); @@ -695,7 +698,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (visibilityTags != null) { labelCache.put(labelsExp, visibilityTags); List updatedCells = new ArrayList(); - for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) { + for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { Cell cell = cellScanner.current(); List tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); @@ -708,18 +711,102 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb cell.getValueLength(), tags); updatedCells.add(updatedCell); } - p.getFamilyCellMap().clear(); - // Clear and add new Cells to the Mutation. - for (Cell cell : updatedCells) { + m.getFamilyCellMap().clear(); + // Clear and add new Cells to the Mutation. + for (Cell cell : updatedCells) { + if (m instanceof Put) { + Put p = (Put) m; p.add(cell); + } else if (m instanceof Delete) { + // Should we fail any delete without CellVisibility? + Delete d = (Delete) m; + d.delete(cell); } } } } - } else if (cellVisibility != null) { - // CellVisibility in a Delete is not legal! Fail the operation - miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE, - "CellVisibility cannot be set on Delete mutation")); + } + } + } + + @Override + public void preDelete(ObserverContext env, Delete delete, + WALEdit edit, Durability durability) throws IOException { + // Here we have two cases + // Case 1: Delete column with specified ts + // If the cell's ts matches with the specified ts in the delete Column cell, + // we have found a match. + // Case 2 : Delete column with latest ts + // Now the issue here is, in the HRegion.batchMutate(), in + // prepareDeleteTimestamps() + // the timestamp for the delete is updated with the latest timestamp. + // If suppose the latest version of the cell does not have the required + // visibility expression + // we should go ahead with the next latest version of the cell where we + // could match the visibility + // expression. + CellVisibility cellVisibility = null; + try { + cellVisibility = delete.getCellVisibility(); + } catch (DeserializationException de) { + throw new RuntimeException("Invalid cell visibility specified " + delete); + } + for (CellScanner cellScanner = delete.cellScanner(); cellScanner.advance();) { + if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { + throw new RuntimeException("Reserved visibility type used " + cellVisibility.toString()); + } + } + List visibilityTags = new ArrayList(); + if (cellVisibility != null) { + String labelsExp = cellVisibility.getExpression(); + try { + visibilityTags = createVisibilityTags(labelsExp, true); + } catch (ParseException e) { + throw new RuntimeException("Invalid cell visibility expression " + labelsExp); + } catch (InvalidLabelException e) { + throw new RuntimeException("Invalid cell visibility specified " + labelsExp); + } + } + for (Map.Entry> e : delete.getFamilyCellMap().entrySet()) { + byte[] family = e.getKey(); + List cells = e.getValue(); + Map kvCount = new TreeMap(Bytes.BYTES_COMPARATOR); + boolean matchFound = false; + boolean isDeleteLatest = false; + for (Cell cell : cells) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + // Check if time is LATEST, change to time of most recent addition if so + // This is expensive. + if (kv.isLatestTimestamp() && kv.isDeleteType()) { + isDeleteLatest = true; + byte[] qual = kv.getQualifier(); + if (qual == null) + qual = HConstants.EMPTY_BYTE_ARRAY; + Get get = new Get(kv.getRow()); + // Fetch all the max versions possible + get.setMaxVersions(); + + get.addColumn(family, qual); + List results = new ArrayList(); + RegionScanner scanner = null; + try { + Scan scan = new Scan(get); + scanner = env.getEnvironment().getRegion().getScanner(scan); + scanner.next(results); + } finally { + if (scanner != null) + scanner.close(); + } + for (Cell getkv : results) { + matchFound = VisibilityUtils.checkForMatchingVisibilityTags(getkv, visibilityTags); + if (matchFound) { + KeyValue getKeyVal = KeyValueUtil.ensureKeyValue(getkv); + Bytes.putBytes(kv.getBuffer(), kv.getTimestampOffset(), getKeyVal.getBuffer(), + getKeyVal.getTimestampOffset(), Bytes.SIZEOF_LONG); + break; + } + } + } } } } @@ -822,36 +909,55 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb return true; } - private List createVisibilityTags(String visibilityLabelsExp) throws IOException, - ParseException, InvalidLabelException { + private List createVisibilityTags(String visibilityLabelsExp, boolean isDelete) + throws IOException, ParseException, InvalidLabelException { ExpressionNode node = null; node = this.expressionParser.parse(visibilityLabelsExp); node = this.expressionExpander.expand(node); List tags = new ArrayList(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); + List labelOrdinals = new ArrayList(); + // Add new tag type + if (!isDelete) { + tags.add(new Tag(VisibilityUtils.DELETE_VISIBILITY_TYPE_FOUND, HConstants.EMPTY_BYTE_ARRAY)); + } if (node.isSingleNode()) { - writeLabelOrdinalsToStream(node, dos); + getLabelOrdinals(node, labelOrdinals); + writeLabelOrdinalsToStream(labelOrdinals, dos); tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); baos.reset(); } else { NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node; if (nlNode.getOperator() == Operator.OR) { for (ExpressionNode child : nlNode.getChildExps()) { - writeLabelOrdinalsToStream(child, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + getLabelOrdinals(child, labelOrdinals); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, + baos.toByteArray())); baos.reset(); + labelOrdinals.clear(); } } else { - writeLabelOrdinalsToStream(nlNode, dos); - tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray())); + getLabelOrdinals(nlNode, labelOrdinals); + writeLabelOrdinalsToStream(labelOrdinals, dos); + tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, + baos.toByteArray())); baos.reset(); } } return tags; } - private void writeLabelOrdinalsToStream(ExpressionNode node, DataOutputStream dos) + private void writeLabelOrdinalsToStream(List labelOrdinals, DataOutputStream dos) + throws IOException { + Collections.sort(labelOrdinals); + for (Integer labelOrdinal : labelOrdinals) { + StreamUtils.writeRawVInt32(dos, labelOrdinal); + } + } + + private void getLabelOrdinals(ExpressionNode node, List labelOrdinals) throws IOException, InvalidLabelException { if (node.isSingleNode()) { String identifier = null; @@ -874,11 +980,11 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb if (labelOrdinal == 0) { throw new InvalidLabelException("Invalid visibility label " + identifier); } - StreamUtils.writeRawVInt32(dos, labelOrdinal); + labelOrdinals.add(labelOrdinal); } else { List childExps = ((NonLeafExpressionNode) node).getChildExps(); for (ExpressionNode child : childExps) { - writeLabelOrdinalsToStream(child, dos); + getLabelOrdinals(child, labelOrdinals); } } } @@ -920,6 +1026,17 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } @Override + public DeleteTracker postInstantiateDeleteTracker( + ObserverContext ctx, DeleteTracker delTracker) + throws IOException { + HRegion region = ctx.getEnvironment().getRegion(); + TableName table = region.getRegionInfo().getTable(); + if (table.isSystemTable() || table.equals(LABELS_TABLE_NAME)) { + return delTracker; + } + return new VisibilityScanDeleteTracker(); + } + @Override public RegionScanner postScannerOpen(final ObserverContext c, final Scan scan, final RegionScanner s) throws IOException { User user = getActiveUser(); @@ -1097,7 +1214,8 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb } } try { - tags.addAll(createVisibilityTags(cellVisibility.getExpression())); + tags.addAll(createVisibilityTags(cellVisibility.getExpression(), + mutation instanceof Delete)); } catch (ParseException e) { throw new IOException(e); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java new file mode 100644 index 0000000..8ee7475 --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java @@ -0,0 +1,270 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Similar to ScanDeletTracker but tracks the visibility expression also before + * deciding if a Cell can be considered deleted + */ +public class VisibilityScanDeleteTracker extends ScanDeleteTracker { + + // Its better to track the visibility tags in delete based on each type. Create individual + // data structures for tracking each of them. This would ensure that there is no tracking based + // on time and also would handle all cases where deletefamily or deletecolumns is specified with + // Latest_timestamp. In such cases the ts in the delete marker and the masking + // put will not be same. So going with individual data structures for different delete + // type would solve this problem and also ensure that the combination of different type + // of deletes with diff ts would also work fine + private List visibilityTagsDeleteFamily; + private List visibilityTagsDeleteFamilyVersion; + private List visibilityTagsDeleteColumns; + private List visiblityTagsDeleteColumn; + + public VisibilityScanDeleteTracker() { + super(); + } + + @Override + public void add(Cell delCell) { + //Cannot call super.add because need to find if the delete needs to be considered + long timestamp = delCell.getTimestamp(); + int qualifierOffset = delCell.getQualifierOffset(); + int qualifierLength = delCell.getQualifierLength(); + byte type = delCell.getTypeByte(); + if (!hasFamilyStamp || timestamp > familyStamp) { + if (type == KeyValue.Type.DeleteFamily.getCode()) { + hasFamilyStamp = true; + familyStamp = timestamp; + extractDeleteTags(delCell, KeyValue.Type.DeleteFamily); + return; + } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) { + familyVersionStamps.add(timestamp); + extractDeleteTags(delCell, KeyValue.Type.DeleteFamilyVersion); + return; + } + + if (deleteBuffer != null && type < deleteType) { + // same column, so ignore less specific delete + if (Bytes.equals(deleteBuffer, deleteOffset, deleteLength, + delCell.getQualifierArray(), qualifierOffset, qualifierLength)){ + return; + } + } + // new column, or more general delete type + deleteBuffer = delCell.getQualifierArray(); + deleteOffset = qualifierOffset; + deleteLength = qualifierLength; + deleteType = type; + deleteTimestamp = timestamp; + extractDeleteTags(delCell, KeyValue.Type.codeToType(type)); + } + } + + private void extractDeleteTags(Cell delCell, Type type) { + // If tag is present in the delete + if (delCell.getTagsLength() > 0) { + switch (type) { + case DeleteFamily: + visibilityTagsDeleteFamily = new ArrayList(); + VisibilityUtils.getVisibilityTags(delCell, visibilityTagsDeleteFamily); + if(visibilityTagsDeleteFamily.isEmpty()) { + visibilityTagsDeleteFamily = null; + } + break; + case DeleteFamilyVersion: + visibilityTagsDeleteFamilyVersion = new ArrayList(); + VisibilityUtils.getVisibilityTags(delCell, visibilityTagsDeleteFamilyVersion); + if(visibilityTagsDeleteFamilyVersion.isEmpty()) { + visibilityTagsDeleteFamilyVersion = null; + } + break; + case DeleteColumn: + visibilityTagsDeleteColumns = new ArrayList(); + VisibilityUtils.getVisibilityTags(delCell, visibilityTagsDeleteColumns); + if(visibilityTagsDeleteColumns.isEmpty()) { + visibilityTagsDeleteColumns = null; + } + break; + case Delete: + visiblityTagsDeleteColumn = new ArrayList(); + VisibilityUtils.getVisibilityTags(delCell, visiblityTagsDeleteColumn); + if(visiblityTagsDeleteColumn.isEmpty()) { + visiblityTagsDeleteColumn = null; + } + break; + + } + } else { + switch (type) { + case DeleteFamily: + visibilityTagsDeleteFamily = null; + break; + case DeleteFamilyVersion: + visibilityTagsDeleteFamilyVersion = null; + break; + case DeleteColumn: + visibilityTagsDeleteColumns = null; + break; + case Delete: + visiblityTagsDeleteColumn = null; + break; + + } + } + } + + @Override + public DeleteResult isDeleted(Cell cell) { + long timestamp = cell.getTimestamp(); + int qualifierOffset = cell.getQualifierOffset(); + int qualifierLength = cell.getQualifierLength(); + if (hasFamilyStamp && timestamp <= familyStamp) { + if (visibilityTagsDeleteFamily != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsDeleteFamily); + if (matchFound) { + // Should we return FAMILY_VERSION_DELETED or FAMILY_DELETED here + return DeleteResult.FAMILY_VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + + } else { + List tags = new ArrayList(); + VisibilityUtils.getVisibilityTags(cell, tags); + if (tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.FAMILY_DELETED; + } + } + } + + if (familyVersionStamps.contains(Long.valueOf(timestamp))) { + if (visibilityTagsDeleteFamilyVersion != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsDeleteFamilyVersion); + if (matchFound) { + return DeleteResult.FAMILY_VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + + } else { + List tags = new ArrayList(); + VisibilityUtils.getVisibilityTags(cell, tags); + if (tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.FAMILY_VERSION_DELETED; + } + } + } + + if (deleteBuffer != null) { + int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, cell.getQualifierArray(), + qualifierOffset, qualifierLength); + + if (ret == 0) { + if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { + if (visibilityTagsDeleteColumns != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visibilityTagsDeleteColumns); + if (matchFound) { + // Should we return COLUMN_DELETED or VERSION_DELETED + return DeleteResult.VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + + } else { + List tags = new ArrayList(); + VisibilityUtils.getVisibilityTags(cell, tags); + if (tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.COLUMN_DELETED; + } + } + } + // Delete (aka DeleteVersion) + // If the timestamp is the same, keep this one + if (timestamp == deleteTimestamp) { + if (visiblityTagsDeleteColumn != null) { + boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell, + visiblityTagsDeleteColumn); + if (matchFound) { + return DeleteResult.VERSION_DELETED; + } else { + return DeleteResult.NOT_DELETED; + } + } else { + List tags = new ArrayList(); + VisibilityUtils.getVisibilityTags(cell, tags); + if (tags.size() > 0) { + return DeleteResult.NOT_DELETED; + } else { + // No tags + return DeleteResult.VERSION_DELETED; + } + } + } + // use assert or not? + assert timestamp < deleteTimestamp; + + // different timestamp, let's clear the buffer. + deleteBuffer = null; + visiblityTagsDeleteColumn = null; + } else if (ret < 0) { + // Next column case. + deleteBuffer = null; + visibilityTagsDeleteColumns = null; + visiblityTagsDeleteColumn = null; + } else { + throw new IllegalStateException("isDelete failed: deleteBuffer=" + + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier=" + + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength) + + ", timestamp=" + timestamp + ", comparison result: " + ret); + } + } + return DeleteResult.NOT_DELETED; + } + + @Override + public void reset() { + super.reset(); + visibilityTagsDeleteColumns = null; + visibilityTagsDeleteFamily = null; + visibilityTagsDeleteFamilyVersion = null; + visiblityTagsDeleteColumn = null; + } +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java index 0c7764f..0bd55a6 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java @@ -19,25 +19,32 @@ package org.apache.hadoop.hbase.security.visibility; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import com.google.protobuf.HBaseZeroCopyByteString; - import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.util.ReflectionUtils; +import com.google.protobuf.HBaseZeroCopyByteString; import com.google.protobuf.InvalidProtocolBufferException; /** @@ -46,9 +53,10 @@ import com.google.protobuf.InvalidProtocolBufferException; @InterfaceAudience.Private public class VisibilityUtils { - public static final String VISIBILITY_LABEL_GENERATOR_CLASS = + public static final String VISIBILITY_LABEL_GENERATOR_CLASS = "hbase.regionserver.scan.visibility.label.generator.class"; public static final byte VISIBILITY_TAG_TYPE = TagType.VISIBILITY_TAG_TYPE; + public static final byte DELETE_VISIBILITY_TYPE_FOUND = TagType.DELETE_VISIBILITY_TYPE_FOUND; public static final String SYSTEM_LABEL = "system"; private static final String COMMA = ","; @@ -156,4 +164,138 @@ public class VisibilityUtils { } return slgs; } + + public static boolean getVisibilityTags(Cell cell, List tags) { + boolean sortedOrder = false; + byte[] tagBytes = CellUtil.getTagArray(cell); + Iterator tagsIterator = CellUtil.tagsIterator(tagBytes, 0, tagBytes.length); + while (tagsIterator.hasNext()) { + Tag tag = tagsIterator.next(); + if(tag.getType() == VisibilityUtils.DELETE_VISIBILITY_TYPE_FOUND) { + sortedOrder = true; + continue; + } + if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { + tags.add(tag); + } + } + return sortedOrder; + } + + public static boolean checkForMatchingVisibilityTags(Cell cell, + List visibilityTagsInDeleteCell) { + List tags = new ArrayList(); + boolean sortedTags = getVisibilityTags(cell, tags); + if (tags.size() == 0) { + // Early out if there are no tags in the cell + return false; + } + if (sortedTags) { + return checkForMatchingVisibilityTagsWithSortedOrder(cell.getTagsLength(), + visibilityTagsInDeleteCell, tags); + } else { + try { + return checkForMatchingVisibilityTagsWithOutSortedOrder(cell, visibilityTagsInDeleteCell); + } catch (Exception e) { + // Should not happen + throw new RuntimeException("Exception while sorting the tags from the cell"); + } + } + } + + private static boolean checkForMatchingVisibilityTagsWithOutSortedOrder(Cell cell, + List visibilityTagsInDeleteCell) throws IOException { + List> sortedDeleteTags = sortTagsBasedOnOrdinal( + visibilityTagsInDeleteCell); + List> sortedTags = sortedTagsBasedOnOrdinal(cell); + return compareTagsOrdinals(sortedDeleteTags, sortedTags); + } + + private static boolean checkForMatchingVisibilityTagsWithSortedOrder(short tagsLength, + List visibilityTagsInDeleteCell, List tags) { + boolean matchFound = false; + if (tagsLength > 0) { + // the tags in visibility delete cell will also include the new type of tag + if ((visibilityTagsInDeleteCell.size()) != tags.size()) { + // If the size does not match. Definitely we are not comparing the equal + // tags. + // Return false in that case. + return matchFound; + } + for (Tag tag : visibilityTagsInDeleteCell) { + matchFound = false; + for (Tag givenTag : tags) { + if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(), + givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) { + matchFound = true; + break; + } + } + } + } + return matchFound; + } + + private static List> sortedTagsBasedOnOrdinal(Cell cell) + throws IOException { + Iterator tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), + cell.getTagsLength()); + List> fullTagsList = new ArrayList>(); + List tagsOrdinalInSortedOrder = new ArrayList(); + while (tagsItr.hasNext()) { + Tag tag = tagsItr.next(); + getSortedTagOrdinals(fullTagsList, tagsOrdinalInSortedOrder, tag); + } + return fullTagsList; + } + + private static List> sortTagsBasedOnOrdinal(List tags) + throws IOException { + List> fullTagsList = new ArrayList>(); + List tagsOrdinalInSortedOrder = new ArrayList(); + for (Tag tag : tags) { + getSortedTagOrdinals(fullTagsList, tagsOrdinalInSortedOrder, tag); + } + return fullTagsList; + } + + private static void getSortedTagOrdinals(List> fullTagsList, + List tagsOrdinalInSortedOrder, Tag tag) throws IOException { + if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) { + int offset = tag.getTagOffset(); + int endOffset = offset + tag.getTagLength(); + while (offset < endOffset) { + Pair result = StreamUtils.readRawVarint32(tag.getBuffer(), offset); + tagsOrdinalInSortedOrder.add(result.getFirst()); + offset += result.getSecond(); + } + Collections.sort(tagsOrdinalInSortedOrder); + fullTagsList.add(cloneList(tagsOrdinalInSortedOrder)); + tagsOrdinalInSortedOrder.clear(); + } + } + + private static boolean compareTagsOrdinals(List> tagsInDeletes, + List> tags) { + boolean matchFound = false; + if (tagsInDeletes.size() != tags.size()) { + return matchFound; + } else { + for (List deleteTagOrdinals : tagsInDeletes) { + matchFound = false; + for (List tagOrdinals : tags) { + if (deleteTagOrdinals.containsAll(tagOrdinals)) { + matchFound = true; + break; + } + } + } + return matchFound; + } + } + + private static List cloneList(List tagsOrdinalList) { + List clone = new LinkedList(tagsOrdinalList); + return clone; + } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java new file mode 100644 index 0000000..72cdcbd --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java @@ -0,0 +1,1784 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +/** + * Tests visibility labels with deletes + */ +@Category(MediumTests.class) +public class TestVisibilityLabelsWithDeletes { + private static final String TOPSECRET = "TOPSECRET"; + private static final String PUBLIC = "PUBLIC"; + private static final String PRIVATE = "PRIVATE"; + private static final String CONFIDENTIAL = "CONFIDENTIAL"; + private static final String SECRET = "SECRET"; + public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final byte[] row1 = Bytes.toBytes("row1"); + private static final byte[] row2 = Bytes.toBytes("row2"); + private final static byte[] fam = Bytes.toBytes("info"); + private final static byte[] qual = Bytes.toBytes("qual"); + private final static byte[] qual1 = Bytes.toBytes("qual1"); + private final static byte[] qual2 = Bytes.toBytes("qual2"); + private final static byte[] value = Bytes.toBytes("value"); + public static Configuration conf; + + @Rule + public final TestName TEST_NAME = new TestName(); + public static User SUPERUSER; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // setup configuration + conf = TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); + conf.setBoolean("hbase.online.schema.update.enable", true); + conf.setInt("hfile.format.version", 3); + conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName()); + conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName()); + conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, + ScanLabelGenerator.class); + conf.set("hbase.superuser", "admin"); + TEST_UTIL.startMiniCluster(2); + SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); + + // Wait for the labels table to become available + TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000); + addLabels(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testVisibilityLabelsWithDeleteColumns() throws Throwable { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + TOPSECRET, + SECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(TOPSECRET + "&" + SECRET)); + d.deleteColumns(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteFamily() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET, CONFIDENTIAL + "|" + + TOPSECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row2); + d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL)); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteFamilyVersion() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + long[] ts = new long[] { 123l, 125l }; + final HTable table = createTableAndWriteDataWithLabels(tableName, ts, CONFIDENTIAL + "|" + + TOPSECRET, SECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL)); + d.deleteFamilyVersion(fam, 123l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteColumnExactVersion() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET }, + SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + VisibilityLabelsResponse response = SUPERUSER.runAs(action); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + long[] ts = new long[] { 123l, 125l }; + final HTable table = createTableAndWriteDataWithLabels(tableName, ts, CONFIDENTIAL + "|" + + TOPSECRET, SECRET); + try { + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL)); + d.deleteColumn(fam, qual, 123l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityLabelsWithDeleteColumnsWithMultipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteColumns(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + private HTable doPuts(TableName tableName) throws IOException, InterruptedIOException, + RetriesExhaustedWithDetailsException, InterruptedException { + HTable table; + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 125l, value); + put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 126l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + put = new Put(Bytes.toBytes("row2")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + TOPSECRET + + "&" + SECRET + ")")); + table.put(put); + return table; + } + + private HTable doPutsWithDiffCols(TableName tableName) throws IOException, + InterruptedIOException, RetriesExhaustedWithDetailsException, InterruptedException { + HTable table; + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 125l, value); + put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual1, 126l, value); + put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual2, 127l, value); + put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + table.put(put); + return table; + } + + private HTable doPutsWithoutVisibility(TableName tableName) throws IOException, + InterruptedIOException, RetriesExhaustedWithDetailsException, InterruptedException { + HTable table; + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 125l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 126l, value); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 127l, value); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + put = new Put(Bytes.toBytes("row2")); + put.add(fam, qual, 127l, value); + table.put(put); + return table; + } + + + @Test + public void testDeleteColumnWithSpecificTimeStampUsingMultipleVersionsUnMatchingVisExpression() + throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteColumn(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnWithLatestTimeStampUsingMultipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnWithLatestTimeStampUsingMultipleVersionsAfterCompaction() + throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Put put = new Put(Bytes.toBytes("row3")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE)); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + TEST_UTIL.getHBaseAdmin().majorCompact(tableName.getNameAsString()); + // Sleep to ensure compaction happens. Need to do it in a better way + Thread.sleep(5000); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 3); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilyLatestTimeStampWithMulipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnswithMultipleColumnsWithMultipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPutsWithDiffCols(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumns(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(), + current.getQualifierLength(), qual1, 0, qual1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(), + current.getQualifierLength(), qual2, 0, qual2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilyWithoutCellVisibilityWithMulipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPutsWithoutVisibility(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + // All cells wrt row1 should be deleted as we are not passing the Cell Visibility + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilyLatestTimeStampWithMulipleVersionsWithoutCellVisibilityInPuts() + throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPutsWithoutVisibility(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteFamily(fam); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + @Test + public void testDeleteFamilySpecificTimeStampWithMulipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteFamily(fam, 126l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(6); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testScanAfterCompaction() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + SECRET + "&" + TOPSECRET+")")); + d.deleteFamily(fam, 126l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Put put = new Put(Bytes.toBytes("row3")); + put.add(fam, qual, 127l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE)); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + TEST_UTIL.getHBaseAdmin().compact(tableName.getNameAsString()); + Thread.sleep(5000); + // Sleep to ensure compaction happens. Need to do it in a better way + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 3); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteFamilySpecificTimeStampWithMulipleVersionsDoneTwice() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteFamily(fam, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteFamily(fam, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + assertEquals(current.getTimestamp(), 127l); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnSpecificTimeStampWithMulipleVersionsDoneTwice() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteColumn(fam, qual, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + assertEquals(current.getTimestamp(), 127l); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnSpecificTimeStampWithMulipleVersionsDoneTwice1() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")" + + "|(" + TOPSECRET + "&" + SECRET + ")")); + d.deleteColumn(fam, qual, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + assertEquals(current.getTimestamp(), 127l); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testDeleteColumnAndDeleteFamilylSpecificTimeStampWithMulipleVersion() + throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET)); + d.deleteColumn(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteFamily(fam, 124l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + assertEquals(current.getTimestamp(), 127l); + } finally { + if (table != null) { + table.close(); + } + } + } + + private void setAuths() throws IOException, InterruptedException { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + try { + return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET, + TOPSECRET }, SUPERUSER.getShortName()); + } catch (Throwable e) { + } + return null; + } + }; + SUPERUSER.runAs(action); + } + + @Test + public void testDiffDeleteTypesForTheSameCellUsingMultipleVersions() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + // Do not flush here. + table = doPuts(tableName); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteColumns(fam, qual, 125l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + + // Issue 2nd delete + actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + + TOPSECRET + "&" + SECRET+")")); + d.deleteColumn(fam, qual, 127l); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + scanner = table.getScanner(s); + next = scanner.next(3); + assertTrue(next.length == 2); + cellScanner = next[0].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + @Test + public void testDeleteColumnLatestWithNoCellVisibility() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + table = doPuts(tableName); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteColumn(fam, qual); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 2); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 127l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 126l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 125l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + cellScanner = next[1].cellScanner(); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row2, 0, row2.length)); + } finally { + if (table != null) { + table.close(); + } + } + } + + @Test + public void testVisibilityExpressionWithNotEqualORCondition() throws Exception { + setAuths(); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = null; + try { + HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + table = new HTable(conf, tableName); + Put put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 123l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + table.put(put); + put = new Put(Bytes.toBytes("row1")); + put.add(fam, qual, 124l, value); + put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "|" + PRIVATE)); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() { + public Void run() throws Exception { + try { + HTable table = new HTable(conf, TEST_NAME.getMethodName()); + Delete d = new Delete(row1); + d.deleteColumn(fam, qual, 124l); + d.setCellVisibility(new CellVisibility(PRIVATE )); + table.delete(d); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(actiona); + + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + Scan s = new Scan(); + s.setMaxVersions(5); + s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET)); + ResultScanner scanner = table.getScanner(s); + Result[] next = scanner.next(3); + assertTrue(next.length == 1); + CellScanner cellScanner = next[0].cellScanner(); + cellScanner.advance(); + Cell current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 124l); + cellScanner.advance(); + current = cellScanner.current(); + assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), + current.getRowLength(), row1, 0, row1.length)); + assertEquals(current.getTimestamp(), 123l); + } finally { + if (table != null) { + table.close(); + } + } + } + + public static HTable createTableAndWriteDataWithLabels(TableName tableName, String... labelExps) + throws Exception { + HTable table = null; + try { + table = TEST_UTIL.createTable(tableName, fam); + int i = 1; + List puts = new ArrayList(); + for (String labelExp : labelExps) { + Put put = new Put(Bytes.toBytes("row" + i)); + put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value); + put.setCellVisibility(new CellVisibility(labelExp)); + puts.add(put); + table.put(put); + i++; + } + // table.put(puts); + } finally { + if (table != null) { + table.close(); + } + } + return table; + } + + public static HTable createTableAndWriteDataWithLabels(TableName tableName, long[] timestamp, + String... labelExps) throws Exception { + HTable table = null; + try { + table = TEST_UTIL.createTable(tableName, fam); + int i = 1; + List puts = new ArrayList(); + for (String labelExp : labelExps) { + Put put = new Put(Bytes.toBytes("row" + i)); + put.add(fam, qual, timestamp[i - 1], value); + put.setCellVisibility(new CellVisibility(labelExp)); + puts.add(put); + table.put(put); + TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString()); + i++; + } + } finally { + if (table != null) { + table.close(); + } + } + return table; + } + + public static void addLabels() throws Exception { + PrivilegedExceptionAction action = + new PrivilegedExceptionAction() { + public VisibilityLabelsResponse run() throws Exception { + String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE }; + try { + VisibilityClient.addLabels(conf, labels); + } catch (Throwable t) { + throw new IOException(t); + } + return null; + } + }; + SUPERUSER.runAs(action); + } +}