();
+ protected byte [] deleteBuffer = null;
+ protected int deleteOffset = 0;
+ protected int deleteLength = 0;
+ protected byte deleteType = 0;
+ protected long deleteTimestamp = 0L;
/**
* Constructor for ScanDeleteTracker
@@ -65,7 +65,7 @@ public class ScanDeleteTracker implements DeleteTracker {
* Add the specified KeyValue to the list of deletes to check against for
* this row operation.
*
- * This is called when a Delete is encountered in a StoreFile.
+ * This is called when a Delete is encountered.
* @param cell - the delete cell
*/
@Override
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
index a152fcc..c76cbcf 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
@@ -30,6 +30,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -75,6 +76,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.util.StreamUtils;
@@ -93,6 +95,7 @@ import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.Visibil
import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService;
import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.DeleteTracker;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -688,10 +691,8 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb
new OperationStatus(SANITY_CHECK_FAILURE, de.getMessage()));
continue;
}
- if (m instanceof Put) {
- Put p = (Put) m;
boolean sanityFailure = false;
- for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) {
+ for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {
if (!checkForReservedVisibilityTagPresence(cellScanner.current())) {
miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE,
"Mutation contains cell with reserved type tag"));
@@ -717,7 +718,7 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb
if (visibilityTags != null) {
labelCache.put(labelsExp, visibilityTags);
List updatedCells = new ArrayList();
- for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) {
+ for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {
Cell cell = cellScanner.current();
List tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
cell.getTagsLength());
@@ -730,22 +731,68 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb
cell.getValueLength(), tags);
updatedCells.add(updatedCell);
}
- p.getFamilyCellMap().clear();
- // Clear and add new Cells to the Mutation.
- for (Cell cell : updatedCells) {
+ m.getFamilyCellMap().clear();
+ // Clear and add new Cells to the Mutation.
+ for (Cell cell : updatedCells) {
+ if (m instanceof Put) {
+ Put p = (Put) m;
p.add(cell);
+ } else if (m instanceof Delete) {
+ // Should we fail any delete without CellVisibility?
+ Delete d = (Delete) m;
+ d.addDeleteMarker(cell);
}
}
}
}
- } else if (cellVisibility != null) {
- // CellVisibility in a Delete is not legal! Fail the operation
- miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE,
- "CellVisibility cannot be set on Delete mutation"));
}
}
}
+
+ @Override
+ public void prePrepareTimeStampForDeleteVersion(
+ ObserverContext ctx, Mutation delete, Cell cell,
+ byte[] byteNow, Get get) throws IOException {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ CellVisibility cellVisibility = null;
+ try {
+ cellVisibility = delete.getCellVisibility();
+ } catch (DeserializationException de) {
+ throw new RuntimeException("Invalid cell visibility specified " + delete);
+ }
+ for (CellScanner cellScanner = delete.cellScanner(); cellScanner.advance();) {
+ if (!checkForReservedVisibilityTagPresence(cellScanner.current())) {
+ throw new RuntimeException("Reserved visibility type used " + cellVisibility.toString());
+ }
+ }
+ List visibilityTags = new ArrayList();
+ if (cellVisibility != null) {
+ String labelsExp = cellVisibility.getExpression();
+ try {
+ visibilityTags = createVisibilityTags(labelsExp);
+ } catch (ParseException e) {
+ throw new RuntimeException("Invalid cell visibility expression " + labelsExp);
+ } catch (InvalidLabelException e) {
+ throw new RuntimeException("Invalid cell visibility specified " + labelsExp);
+ }
+ }
+ get.setFilter(new DeleteVersionVisibilityExpressionFilter(visibilityTags));
+ List result = ctx.getEnvironment().getRegion().get(get, false);
+ if (result.size() < get.getMaxVersions()) {
+ // Nothing to delete
+ kv.updateLatestStamp(Bytes.toBytes(Long.MIN_VALUE));
+ return;
+ }
+ if (result.size() > get.getMaxVersions()) {
+ throw new RuntimeException("Unexpected size: " + result.size());
+ }
+ KeyValue getkv = KeyValueUtil.ensureKeyValue(result.get(get.getMaxVersions() - 1));
+ Bytes.putBytes(kv.getBuffer(), kv.getTimestampOffset(), getkv.getBuffer(),
+ getkv.getTimestampOffset(), Bytes.SIZEOF_LONG);
+ ctx.bypass();
+ }
+
@Override
public void postBatchMutate(ObserverContext c,
MiniBatchOperationInProgress miniBatchOp) throws IOException {
@@ -844,36 +891,54 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb
return true;
}
- private List createVisibilityTags(String visibilityLabelsExp) throws IOException,
- ParseException, InvalidLabelException {
+ private List createVisibilityTags(String visibilityLabelsExp)
+ throws IOException, ParseException, InvalidLabelException {
ExpressionNode node = null;
node = this.expressionParser.parse(visibilityLabelsExp);
node = this.expressionExpander.expand(node);
List tags = new ArrayList();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
+ List labelOrdinals = new ArrayList();
+ // Add new tag type
+ tags.add(new Tag(VisibilityUtils.VISBILITY_EXP_SERIALIZATION_FORMAT,
+ HConstants.EMPTY_BYTE_ARRAY));
if (node.isSingleNode()) {
- writeLabelOrdinalsToStream(node, dos);
+ getLabelOrdinals(node, labelOrdinals);
+ writeLabelOrdinalsToStream(labelOrdinals, dos);
tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
baos.reset();
} else {
NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node;
if (nlNode.getOperator() == Operator.OR) {
for (ExpressionNode child : nlNode.getChildExps()) {
- writeLabelOrdinalsToStream(child, dos);
- tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
+ getLabelOrdinals(child, labelOrdinals);
+ writeLabelOrdinalsToStream(labelOrdinals, dos);
+ tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE,
+ baos.toByteArray()));
baos.reset();
+ labelOrdinals.clear();
}
} else {
- writeLabelOrdinalsToStream(nlNode, dos);
- tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
+ getLabelOrdinals(nlNode, labelOrdinals);
+ writeLabelOrdinalsToStream(labelOrdinals, dos);
+ tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE,
+ baos.toByteArray()));
baos.reset();
}
}
return tags;
}
- private void writeLabelOrdinalsToStream(ExpressionNode node, DataOutputStream dos)
+ private void writeLabelOrdinalsToStream(List labelOrdinals, DataOutputStream dos)
+ throws IOException {
+ Collections.sort(labelOrdinals);
+ for (Integer labelOrdinal : labelOrdinals) {
+ StreamUtils.writeRawVInt32(dos, labelOrdinal);
+ }
+ }
+
+ private void getLabelOrdinals(ExpressionNode node, List labelOrdinals)
throws IOException, InvalidLabelException {
if (node.isSingleNode()) {
String identifier = null;
@@ -896,11 +961,11 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb
if (labelOrdinal == 0) {
throw new InvalidLabelException("Invalid visibility label " + identifier);
}
- StreamUtils.writeRawVInt32(dos, labelOrdinal);
+ labelOrdinals.add(labelOrdinal);
} else {
List childExps = ((NonLeafExpressionNode) node).getChildExps();
for (ExpressionNode child : childExps) {
- writeLabelOrdinalsToStream(child, dos);
+ getLabelOrdinals(child, labelOrdinals);
}
}
}
@@ -942,6 +1007,17 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb
}
@Override
+ public DeleteTracker postInstantiateDeleteTracker(
+ ObserverContext ctx, DeleteTracker delTracker)
+ throws IOException {
+ HRegion region = ctx.getEnvironment().getRegion();
+ TableName table = region.getRegionInfo().getTable();
+ if (table.isSystemTable() || table.equals(LABELS_TABLE_NAME)) {
+ return delTracker;
+ }
+ return new VisibilityScanDeleteTracker();
+ }
+ @Override
public RegionScanner postScannerOpen(final ObserverContext c,
final Scan scan, final RegionScanner s) throws IOException {
User user = getActiveUser();
@@ -1409,4 +1485,22 @@ public class VisibilityController extends BaseRegionObserver implements MasterOb
}
}
}
+
+ static class DeleteVersionVisibilityExpressionFilter extends FilterBase {
+ private List visibilityTags;
+
+ public DeleteVersionVisibilityExpressionFilter(List visibilityTags) {
+ this.visibilityTags = visibilityTags;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell kv) throws IOException {
+ boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(kv, visibilityTags, true);
+ if (matchFound) {
+ return ReturnCode.INCLUDE;
+ } else {
+ return ReturnCode.SKIP;
+ }
+ }
+ }
}
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java
new file mode 100644
index 0000000..e74695f
--- /dev/null
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java
@@ -0,0 +1,272 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.visibility;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Similar to ScanDeletTracker but tracks the visibility expression also before
+ * deciding if a Cell can be considered deleted
+ */
+public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
+
+ // Its better to track the visibility tags in delete based on each type. Create individual
+ // data structures for tracking each of them. This would ensure that there is no tracking based
+ // on time and also would handle all cases where deletefamily or deletecolumns is specified with
+ // Latest_timestamp. In such cases the ts in the delete marker and the masking
+ // put will not be same. So going with individual data structures for different delete
+ // type would solve this problem and also ensure that the combination of different type
+ // of deletes with diff ts would also work fine
+ private List visibilityTagsDeleteFamily;
+ private List visibilityTagsDeleteFamilyVersion;
+ private List visibilityTagsDeleteColumns;
+ private List visiblityTagsDeleteColumn;
+
+ public VisibilityScanDeleteTracker() {
+ super();
+ }
+
+ @Override
+ public void add(Cell delCell) {
+ //Cannot call super.add because need to find if the delete needs to be considered
+ long timestamp = delCell.getTimestamp();
+ int qualifierOffset = delCell.getQualifierOffset();
+ int qualifierLength = delCell.getQualifierLength();
+ byte type = delCell.getTypeByte();
+ if (!hasFamilyStamp || timestamp > familyStamp) {
+ if (type == KeyValue.Type.DeleteFamily.getCode()) {
+ hasFamilyStamp = true;
+ familyStamp = timestamp;
+ extractDeleteTags(delCell, KeyValue.Type.DeleteFamily);
+ return;
+ } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
+ familyVersionStamps.add(timestamp);
+ extractDeleteTags(delCell, KeyValue.Type.DeleteFamilyVersion);
+ return;
+ }
+
+ if (deleteBuffer != null && type < deleteType) {
+ // same column, so ignore less specific delete
+ if (Bytes.equals(deleteBuffer, deleteOffset, deleteLength,
+ delCell.getQualifierArray(), qualifierOffset, qualifierLength)){
+ return;
+ }
+ }
+ // new column, or more general delete type
+ deleteBuffer = delCell.getQualifierArray();
+ deleteOffset = qualifierOffset;
+ deleteLength = qualifierLength;
+ deleteType = type;
+ deleteTimestamp = timestamp;
+ extractDeleteTags(delCell, KeyValue.Type.codeToType(type));
+ }
+ }
+
+ private void extractDeleteTags(Cell delCell, Type type) {
+ // If tag is present in the delete
+ if (delCell.getTagsLength() > 0) {
+ switch (type) {
+ case DeleteFamily:
+ visibilityTagsDeleteFamily = new ArrayList();
+ VisibilityUtils.getVisibilityTags(delCell, visibilityTagsDeleteFamily);
+ if(visibilityTagsDeleteFamily.isEmpty()) {
+ visibilityTagsDeleteFamily = null;
+ }
+ break;
+ case DeleteFamilyVersion:
+ visibilityTagsDeleteFamilyVersion = new ArrayList();
+ VisibilityUtils.getVisibilityTags(delCell, visibilityTagsDeleteFamilyVersion);
+ if(visibilityTagsDeleteFamilyVersion.isEmpty()) {
+ visibilityTagsDeleteFamilyVersion = null;
+ }
+ break;
+ case DeleteColumn:
+ visibilityTagsDeleteColumns = new ArrayList();
+ VisibilityUtils.getVisibilityTags(delCell, visibilityTagsDeleteColumns);
+ if(visibilityTagsDeleteColumns.isEmpty()) {
+ visibilityTagsDeleteColumns = null;
+ }
+ break;
+ case Delete:
+ visiblityTagsDeleteColumn = new ArrayList();
+ VisibilityUtils.getVisibilityTags(delCell, visiblityTagsDeleteColumn);
+ if(visiblityTagsDeleteColumn.isEmpty()) {
+ visiblityTagsDeleteColumn = null;
+ }
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid delete type");
+ }
+ } else {
+ switch (type) {
+ case DeleteFamily:
+ visibilityTagsDeleteFamily = null;
+ break;
+ case DeleteFamilyVersion:
+ visibilityTagsDeleteFamilyVersion = null;
+ break;
+ case DeleteColumn:
+ visibilityTagsDeleteColumns = null;
+ break;
+ case Delete:
+ visiblityTagsDeleteColumn = null;
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid delete type");
+ }
+ }
+ }
+
+ @Override
+ public DeleteResult isDeleted(Cell cell) {
+ long timestamp = cell.getTimestamp();
+ int qualifierOffset = cell.getQualifierOffset();
+ int qualifierLength = cell.getQualifierLength();
+ if (hasFamilyStamp && timestamp <= familyStamp) {
+ if (visibilityTagsDeleteFamily != null) {
+ boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell,
+ visibilityTagsDeleteFamily, false);
+ if (matchFound) {
+ // Should we return FAMILY_VERSION_DELETED or FAMILY_DELETED here
+ return DeleteResult.FAMILY_VERSION_DELETED;
+ } else {
+ return DeleteResult.NOT_DELETED;
+ }
+
+ } else {
+ List tags = new ArrayList();
+ VisibilityUtils.getVisibilityTags(cell, tags);
+ if (tags.size() > 0) {
+ return DeleteResult.NOT_DELETED;
+ } else {
+ // No tags
+ return DeleteResult.FAMILY_DELETED;
+ }
+ }
+ }
+
+ if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
+ if (visibilityTagsDeleteFamilyVersion != null) {
+ boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell,
+ visibilityTagsDeleteFamilyVersion, false);
+ if (matchFound) {
+ return DeleteResult.FAMILY_VERSION_DELETED;
+ } else {
+ return DeleteResult.NOT_DELETED;
+ }
+
+ } else {
+ List tags = new ArrayList();
+ VisibilityUtils.getVisibilityTags(cell, tags);
+ if (tags.size() > 0) {
+ return DeleteResult.NOT_DELETED;
+ } else {
+ // No tags
+ return DeleteResult.FAMILY_VERSION_DELETED;
+ }
+ }
+ }
+
+ if (deleteBuffer != null) {
+ int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, cell.getQualifierArray(),
+ qualifierOffset, qualifierLength);
+
+ if (ret == 0) {
+ if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
+ if (visibilityTagsDeleteColumns != null) {
+ boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell,
+ visibilityTagsDeleteColumns, false);
+ if (matchFound) {
+ // Should we return COLUMN_DELETED or VERSION_DELETED
+ return DeleteResult.VERSION_DELETED;
+ } else {
+ return DeleteResult.NOT_DELETED;
+ }
+
+ } else {
+ List tags = new ArrayList();
+ VisibilityUtils.getVisibilityTags(cell, tags);
+ if (tags.size() > 0) {
+ return DeleteResult.NOT_DELETED;
+ } else {
+ // No tags
+ return DeleteResult.COLUMN_DELETED;
+ }
+ }
+ }
+ // Delete (aka DeleteVersion)
+ // If the timestamp is the same, keep this one
+ if (timestamp == deleteTimestamp) {
+ if (visiblityTagsDeleteColumn != null) {
+ boolean matchFound = VisibilityUtils.checkForMatchingVisibilityTags(cell,
+ visiblityTagsDeleteColumn, false);
+ if (matchFound) {
+ return DeleteResult.VERSION_DELETED;
+ } else {
+ return DeleteResult.NOT_DELETED;
+ }
+ } else {
+ List tags = new ArrayList();
+ VisibilityUtils.getVisibilityTags(cell, tags);
+ if (tags.size() > 0) {
+ return DeleteResult.NOT_DELETED;
+ } else {
+ // No tags
+ return DeleteResult.VERSION_DELETED;
+ }
+ }
+ }
+ // use assert or not?
+ assert timestamp < deleteTimestamp;
+
+ // different timestamp, let's clear the buffer.
+ deleteBuffer = null;
+ visiblityTagsDeleteColumn = null;
+ } else if (ret < 0) {
+ // Next column case.
+ deleteBuffer = null;
+ visibilityTagsDeleteColumns = null;
+ visiblityTagsDeleteColumn = null;
+ } else {
+ throw new IllegalStateException("isDelete failed: deleteBuffer="
+ + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier="
+ + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength)
+ + ", timestamp=" + timestamp + ", comparison result: " + ret);
+ }
+ }
+ return DeleteResult.NOT_DELETED;
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ visibilityTagsDeleteColumns = null;
+ visibilityTagsDeleteFamily = null;
+ visibilityTagsDeleteFamilyVersion = null;
+ visiblityTagsDeleteColumn = null;
+ }
+}
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java
index 0c7764f..1fd30db 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityUtils.java
@@ -19,25 +19,32 @@ package org.apache.hadoop.hbase.security.visibility;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import com.google.protobuf.HBaseZeroCopyByteString;
-
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.TagType;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations;
import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations;
import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.util.ReflectionUtils;
+import com.google.protobuf.HBaseZeroCopyByteString;
import com.google.protobuf.InvalidProtocolBufferException;
/**
@@ -46,9 +53,10 @@ import com.google.protobuf.InvalidProtocolBufferException;
@InterfaceAudience.Private
public class VisibilityUtils {
- public static final String VISIBILITY_LABEL_GENERATOR_CLASS =
+ public static final String VISIBILITY_LABEL_GENERATOR_CLASS =
"hbase.regionserver.scan.visibility.label.generator.class";
public static final byte VISIBILITY_TAG_TYPE = TagType.VISIBILITY_TAG_TYPE;
+ public static final byte VISBILITY_EXP_SERIALIZATION_FORMAT = TagType.VISBILITY_EXP_SERIALIZATION_FORMAT;
public static final String SYSTEM_LABEL = "system";
private static final String COMMA = ",";
@@ -156,4 +164,149 @@ public class VisibilityUtils {
}
return slgs;
}
+
+ public static boolean getVisibilityTags(Cell cell, List tags) {
+ boolean sortedOrder = false;
+ byte[] tagBytes = CellUtil.getTagArray(cell);
+ Iterator tagsIterator = CellUtil.tagsIterator(tagBytes, 0, tagBytes.length);
+ while (tagsIterator.hasNext()) {
+ Tag tag = tagsIterator.next();
+ if(tag.getType() == VisibilityUtils.VISBILITY_EXP_SERIALIZATION_FORMAT) {
+ sortedOrder = true;
+ continue;
+ }
+ if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) {
+ tags.add(tag);
+ }
+ }
+ return sortedOrder;
+ }
+
+ public static boolean checkForMatchingVisibilityTags(Cell cell,
+ List visibilityTagsInDeleteCell, boolean newTagPresentInDelete) {
+ List tags = new ArrayList();
+ boolean sortedTags = getVisibilityTags(cell, tags);
+ if (tags.size() == 0) {
+ // Early out if there are no tags in the cell
+ return false;
+ }
+ if (sortedTags) {
+ return checkForMatchingVisibilityTagsWithSortedOrder(cell.getTagsLength(),
+ visibilityTagsInDeleteCell, tags, newTagPresentInDelete);
+ } else {
+ try {
+ return checkForMatchingVisibilityTagsWithOutSortedOrder(cell, visibilityTagsInDeleteCell);
+ } catch (Exception e) {
+ // Should not happen
+ throw new RuntimeException("Exception while sorting the tags from the cell");
+ }
+ }
+ }
+
+ private static boolean checkForMatchingVisibilityTagsWithOutSortedOrder(Cell cell,
+ List visibilityTagsInDeleteCell) throws IOException {
+ List> sortedDeleteTags = sortTagsBasedOnOrdinal(
+ visibilityTagsInDeleteCell);
+ List> sortedTags = sortedTagsBasedOnOrdinal(cell);
+ return compareTagsOrdinals(sortedDeleteTags, sortedTags);
+ }
+
+ private static boolean checkForMatchingVisibilityTagsWithSortedOrder(short tagsLength,
+ List visibilityTagsInDeleteCell, List tags, boolean newTagPresentInDelete) {
+ boolean matchFound = false;
+ if (tagsLength > 0) {
+ // the tags in visibility delete cell will also include the new type of tag
+ if (newTagPresentInDelete) {
+ if ((visibilityTagsInDeleteCell.size() - 1) != tags.size()) {
+ // If the size does not match. Definitely we are not comparing the
+ // equal
+ // tags.
+ // Return false in that case.
+ return matchFound;
+ }
+ } else {
+ if ((visibilityTagsInDeleteCell.size()) != tags.size()) {
+ // If the size does not match. Definitely we are not comparing the
+ // equal
+ // tags.
+ // Return false in that case.
+ return matchFound;
+ }
+ }
+ for (Tag tag : visibilityTagsInDeleteCell) {
+ matchFound = false;
+ for (Tag givenTag : tags) {
+ if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(),
+ givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) {
+ matchFound = true;
+ break;
+ }
+ }
+ }
+ }
+ return matchFound;
+ }
+
+ private static List> sortedTagsBasedOnOrdinal(Cell cell)
+ throws IOException {
+ Iterator tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
+ cell.getTagsLength());
+ List> fullTagsList = new ArrayList>();
+ List tagsOrdinalInSortedOrder = new ArrayList();
+ while (tagsItr.hasNext()) {
+ Tag tag = tagsItr.next();
+ getSortedTagOrdinals(fullTagsList, tagsOrdinalInSortedOrder, tag);
+ }
+ return fullTagsList;
+ }
+
+ private static List> sortTagsBasedOnOrdinal(List tags)
+ throws IOException {
+ List> fullTagsList = new ArrayList>();
+ List tagsOrdinalInSortedOrder = new ArrayList();
+ for (Tag tag : tags) {
+ getSortedTagOrdinals(fullTagsList, tagsOrdinalInSortedOrder, tag);
+ }
+ return fullTagsList;
+ }
+
+ private static void getSortedTagOrdinals(List> fullTagsList,
+ List tagsOrdinalInSortedOrder, Tag tag) throws IOException {
+ if (tag.getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) {
+ int offset = tag.getTagOffset();
+ int endOffset = offset + tag.getTagLength();
+ while (offset < endOffset) {
+ Pair result = StreamUtils.readRawVarint32(tag.getBuffer(), offset);
+ tagsOrdinalInSortedOrder.add(result.getFirst());
+ offset += result.getSecond();
+ }
+ Collections.sort(tagsOrdinalInSortedOrder);
+ fullTagsList.add(cloneList(tagsOrdinalInSortedOrder));
+ tagsOrdinalInSortedOrder.clear();
+ }
+ }
+
+ private static boolean compareTagsOrdinals(List> tagsInDeletes,
+ List> tags) {
+ boolean matchFound = false;
+ if (tagsInDeletes.size() != tags.size()) {
+ return matchFound;
+ } else {
+ for (List deleteTagOrdinals : tagsInDeletes) {
+ matchFound = false;
+ for (List tagOrdinals : tags) {
+ if (deleteTagOrdinals.containsAll(tagOrdinals)) {
+ matchFound = true;
+ break;
+ }
+ }
+ }
+ return matchFound;
+ }
+ }
+
+ private static List cloneList(List tagsOrdinalList) {
+ List clone = new LinkedList(tagsOrdinalList);
+ return clone;
+ }
}
diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
new file mode 100644
index 0000000..d5d778a
--- /dev/null
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
@@ -0,0 +1,1858 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.visibility;
+
+import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.*;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+/**
+ * Tests visibility labels with deletes
+ */
+@Category(MediumTests.class)
+public class TestVisibilityLabelsWithDeletes {
+ private static final String TOPSECRET = "TOPSECRET";
+ private static final String PUBLIC = "PUBLIC";
+ private static final String PRIVATE = "PRIVATE";
+ private static final String CONFIDENTIAL = "CONFIDENTIAL";
+ private static final String SECRET = "SECRET";
+ public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final byte[] row1 = Bytes.toBytes("row1");
+ private static final byte[] row2 = Bytes.toBytes("row2");
+ private final static byte[] fam = Bytes.toBytes("info");
+ private final static byte[] qual = Bytes.toBytes("qual");
+ private final static byte[] qual1 = Bytes.toBytes("qual1");
+ private final static byte[] qual2 = Bytes.toBytes("qual2");
+ private final static byte[] value = Bytes.toBytes("value");
+ public static Configuration conf;
+
+ @Rule
+ public final TestName TEST_NAME = new TestName();
+ public static User SUPERUSER;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ // setup configuration
+ conf = TEST_UTIL.getConfiguration();
+ conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
+ conf.setBoolean("hbase.online.schema.update.enable", true);
+ conf.setInt("hfile.format.version", 3);
+ conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName());
+ conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName());
+ conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
+ ScanLabelGenerator.class);
+ conf.set("hbase.superuser", "admin");
+ TEST_UTIL.startMiniCluster(2);
+ SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
+
+ // Wait for the labels table to become available
+ TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
+ addLabels();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testVisibilityLabelsWithDeleteColumns() throws Throwable {
+ PrivilegedExceptionAction action =
+ new PrivilegedExceptionAction() {
+ public VisibilityLabelsResponse run() throws Exception {
+ try {
+ return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET },
+ SUPERUSER.getShortName());
+ } catch (Throwable e) {
+ }
+ return null;
+ }
+ };
+ VisibilityLabelsResponse response = SUPERUSER.runAs(action);
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + TOPSECRET,
+ SECRET);
+ try {
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(TOPSECRET + "&" + SECRET));
+ d.deleteColumns(fam, qual);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 1);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testVisibilityLabelsWithDeleteFamily() throws Exception {
+ PrivilegedExceptionAction action =
+ new PrivilegedExceptionAction() {
+ public VisibilityLabelsResponse run() throws Exception {
+ try {
+ return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET },
+ SUPERUSER.getShortName());
+ } catch (Throwable e) {
+ }
+ return null;
+ }
+ };
+ VisibilityLabelsResponse response = SUPERUSER.runAs(action);
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ final HTable table = createTableAndWriteDataWithLabels(tableName, SECRET, CONFIDENTIAL + "|"
+ + TOPSECRET);
+ try {
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row2);
+ d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL));
+ d.deleteFamily(fam);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 1);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testVisibilityLabelsWithDeleteFamilyVersion() throws Exception {
+ PrivilegedExceptionAction action =
+ new PrivilegedExceptionAction() {
+ public VisibilityLabelsResponse run() throws Exception {
+ try {
+ return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET },
+ SUPERUSER.getShortName());
+ } catch (Throwable e) {
+ }
+ return null;
+ }
+ };
+ VisibilityLabelsResponse response = SUPERUSER.runAs(action);
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ long[] ts = new long[] { 123l, 125l };
+ final HTable table = createTableAndWriteDataWithLabels(tableName, ts, CONFIDENTIAL + "|"
+ + TOPSECRET, SECRET);
+ try {
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL));
+ d.deleteFamilyVersion(fam, 123l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 1);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testVisibilityLabelsWithDeleteColumnExactVersion() throws Exception {
+ PrivilegedExceptionAction action =
+ new PrivilegedExceptionAction() {
+ public VisibilityLabelsResponse run() throws Exception {
+ try {
+ return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET },
+ SUPERUSER.getShortName());
+ } catch (Throwable e) {
+ }
+ return null;
+ }
+ };
+ VisibilityLabelsResponse response = SUPERUSER.runAs(action);
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ long[] ts = new long[] { 123l, 125l };
+ final HTable table = createTableAndWriteDataWithLabels(tableName, ts, CONFIDENTIAL + "|"
+ + TOPSECRET, SECRET);
+ try {
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL));
+ d.deleteColumn(fam, qual, 123l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 1);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testVisibilityLabelsWithDeleteColumnsWithMultipleVersions() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPuts(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" +
+ SECRET + "&" + TOPSECRET+")"));
+ d.deleteColumns(fam, qual, 125l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ private HTable doPuts(TableName tableName) throws IOException, InterruptedIOException,
+ RetriesExhaustedWithDetailsException, InterruptedException {
+ HTable table;
+ HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
+ HColumnDescriptor colDesc = new HColumnDescriptor(fam);
+ colDesc.setMaxVersions(5);
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(colDesc);
+ hBaseAdmin.createTable(desc);
+ table = new HTable(conf, tableName);
+ Put put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 123l, value);
+ put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 124l, value);
+ put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 125l, value);
+ put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 126l, value);
+ put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 127l, value);
+ put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ table.put(put);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ put = new Put(Bytes.toBytes("row2"));
+ put.add(fam, qual, 127l, value);
+ put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + TOPSECRET
+ + "&" + SECRET + ")"));
+ table.put(put);
+ return table;
+ }
+
+ private HTable doPutsWithDiffCols(TableName tableName) throws IOException,
+ InterruptedIOException, RetriesExhaustedWithDetailsException, InterruptedException {
+ HTable table;
+ HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
+ HColumnDescriptor colDesc = new HColumnDescriptor(fam);
+ colDesc.setMaxVersions(5);
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(colDesc);
+ hBaseAdmin.createTable(desc);
+ table = new HTable(conf, tableName);
+ Put put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 123l, value);
+ put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 124l, value);
+ put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 125l, value);
+ put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual1, 126l, value);
+ put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual2, 127l, value);
+ put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ table.put(put);
+ return table;
+ }
+
+ private HTable doPutsWithoutVisibility(TableName tableName) throws IOException,
+ InterruptedIOException, RetriesExhaustedWithDetailsException, InterruptedException {
+ HTable table;
+ HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
+ HColumnDescriptor colDesc = new HColumnDescriptor(fam);
+ colDesc.setMaxVersions(5);
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(colDesc);
+ hBaseAdmin.createTable(desc);
+ table = new HTable(conf, tableName);
+ Put put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 123l, value);
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 124l, value);
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 125l, value);
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 126l, value);
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 127l, value);
+ table.put(put);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ put = new Put(Bytes.toBytes("row2"));
+ put.add(fam, qual, 127l, value);
+ table.put(put);
+ return table;
+ }
+
+
+ @Test
+ public void testDeleteColumnWithSpecificTimeStampUsingMultipleVersionsUnMatchingVisExpression()
+ throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPuts(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" +
+ SECRET + "&" + TOPSECRET+")"));
+ d.deleteColumn(fam, qual, 125l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteColumnWithLatestTimeStampUsingMultipleVersions() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPuts(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ d.deleteColumn(fam, qual);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+
+ @Test (timeout=1800000)
+ public void testDeleteColumnWithLatestTimeStampWhenNoVersionMatches() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPuts(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Put put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 128l, value);
+ put.setCellVisibility(new CellVisibility(TOPSECRET));
+ table.put(put);
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(SECRET ));
+ d.deleteColumn(fam, qual);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 128l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 129l, value);
+ put.setCellVisibility(new CellVisibility(SECRET));
+ table.put(put);
+ table.flushCommits();
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ scanner = table.getScanner(s);
+ next = scanner.next(3);
+ assertTrue(next.length == 2);
+ cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 129l);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+ @Test
+ public void testDeleteColumnWithLatestTimeStampUsingMultipleVersionsAfterCompaction()
+ throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPuts(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ d.deleteColumn(fam, qual);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Put put = new Put(Bytes.toBytes("row3"));
+ put.add(fam, qual, 127l, value);
+ put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE));
+ table.put(put);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ TEST_UTIL.getHBaseAdmin().majorCompact(tableName.getNameAsString());
+ // Sleep to ensure compaction happens. Need to do it in a better way
+ Thread.sleep(5000);
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 3);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteFamilyLatestTimeStampWithMulipleVersions() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPuts(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ d.deleteFamily(fam);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteColumnswithMultipleColumnsWithMultipleVersions() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPutsWithDiffCols(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ d.deleteColumns(fam, qual, 125l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 1);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(),
+ current.getQualifierLength(), qual1, 0, qual1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(),
+ current.getQualifierLength(), qual2, 0, qual2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteFamilyWithoutCellVisibilityWithMulipleVersions() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPutsWithoutVisibility(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.deleteFamily(fam);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 1);
+ // All cells wrt row1 should be deleted as we are not passing the Cell Visibility
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteFamilyLatestTimeStampWithMulipleVersionsWithoutCellVisibilityInPuts()
+ throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPutsWithoutVisibility(tableName);
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ d.deleteFamily(fam);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+ @Test
+ public void testDeleteFamilySpecificTimeStampWithMulipleVersions() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPuts(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" +
+ SECRET + "&" + TOPSECRET+")"));
+ d.deleteFamily(fam, 126l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(6);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testScanAfterCompaction() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPuts(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" +
+ SECRET + "&" + TOPSECRET+")"));
+ d.deleteFamily(fam, 126l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Put put = new Put(Bytes.toBytes("row3"));
+ put.add(fam, qual, 127l, value);
+ put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE));
+ table.put(put);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ TEST_UTIL.getHBaseAdmin().compact(tableName.getNameAsString());
+ Thread.sleep(5000);
+ // Sleep to ensure compaction happens. Need to do it in a better way
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 3);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteFamilySpecificTimeStampWithMulipleVersionsDoneTwice() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ // Do not flush here.
+ table = doPuts(tableName);
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ d.deleteFamily(fam, 125l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+
+ // Issue 2nd delete
+ actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ d.deleteFamily(fam, 127l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+ s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ scanner = table.getScanner(s);
+ next = scanner.next(3);
+ assertTrue(next.length == 2);
+ cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ assertEquals(current.getTimestamp(), 127l);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteColumnSpecificTimeStampWithMulipleVersionsDoneTwice() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ // Do not flush here.
+ table = doPuts(tableName);
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ d.deleteColumn(fam, qual, 125l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+
+ // Issue 2nd delete
+ actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ d.deleteColumn(fam, qual, 127l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+ s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ scanner = table.getScanner(s);
+ next = scanner.next(3);
+ assertTrue(next.length == 2);
+ cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ assertEquals(current.getTimestamp(), 127l);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteColumnSpecificTimeStampWithMulipleVersionsDoneTwice1() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ // Do not flush here.
+ table = doPuts(tableName);
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")" +
+ "|(" + TOPSECRET + "&" + SECRET + ")"));
+ d.deleteColumn(fam, qual, 127l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+
+ // Issue 2nd delete
+ actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ d.deleteColumn(fam, qual, 127l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+ s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ scanner = table.getScanner(s);
+ next = scanner.next(3);
+ assertTrue(next.length == 2);
+ cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ assertEquals(current.getTimestamp(), 127l);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteColumnAndDeleteFamilylSpecificTimeStampWithMulipleVersion()
+ throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ // Do not flush here.
+ table = doPuts(tableName);
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
+ d.deleteColumn(fam, qual, 125l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+
+ // Issue 2nd delete
+ actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ d.deleteFamily(fam, 124l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+ s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ scanner = table.getScanner(s);
+ next = scanner.next(3);
+ assertTrue(next.length == 2);
+ cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ assertEquals(current.getTimestamp(), 127l);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ private void setAuths() throws IOException, InterruptedException {
+ PrivilegedExceptionAction action =
+ new PrivilegedExceptionAction() {
+ public VisibilityLabelsResponse run() throws Exception {
+ try {
+ return VisibilityClient.setAuths(conf, new String[] { CONFIDENTIAL, PRIVATE, SECRET,
+ TOPSECRET }, SUPERUSER.getShortName());
+ } catch (Throwable e) {
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(action);
+ }
+
+ @Test
+ public void testDiffDeleteTypesForTheSameCellUsingMultipleVersions() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ // Do not flush here.
+ table = doPuts(tableName);
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ d.deleteColumns(fam, qual, 125l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+
+ // Issue 2nd delete
+ actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ + TOPSECRET + "&" + SECRET+")"));
+ d.deleteColumn(fam, qual, 127l);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+ s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ scanner = table.getScanner(s);
+ next = scanner.next(3);
+ assertTrue(next.length == 2);
+ cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+ @Test
+ public void testDeleteColumnLatestWithNoCellVisibility() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ table = doPuts(tableName);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.deleteColumn(fam, qual);
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 2);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 127l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 126l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 125l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ cellScanner = next[1].cellScanner();
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row2, 0, row2.length));
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
+ public void testVisibilityExpressionWithNotEqualORCondition() throws Exception {
+ setAuths();
+ TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ HTable table = null;
+ try {
+ HBaseAdmin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
+ HColumnDescriptor colDesc = new HColumnDescriptor(fam);
+ colDesc.setMaxVersions(5);
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(colDesc);
+ hBaseAdmin.createTable(desc);
+ table = new HTable(conf, tableName);
+ Put put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 123l, value);
+ put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
+ table.put(put);
+ put = new Put(Bytes.toBytes("row1"));
+ put.add(fam, qual, 124l, value);
+ put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "|" + PRIVATE));
+ table.put(put);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ PrivilegedExceptionAction actiona = new PrivilegedExceptionAction() {
+ public Void run() throws Exception {
+ try {
+ HTable table = new HTable(conf, TEST_NAME.getMethodName());
+ Delete d = new Delete(row1);
+ d.deleteColumn(fam, qual, 124l);
+ d.setCellVisibility(new CellVisibility(PRIVATE ));
+ table.delete(d);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(actiona);
+
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ Scan s = new Scan();
+ s.setMaxVersions(5);
+ s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
+ ResultScanner scanner = table.getScanner(s);
+ Result[] next = scanner.next(3);
+ assertTrue(next.length == 1);
+ CellScanner cellScanner = next[0].cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 124l);
+ cellScanner.advance();
+ current = cellScanner.current();
+ assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
+ current.getRowLength(), row1, 0, row1.length));
+ assertEquals(current.getTimestamp(), 123l);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ public static HTable createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
+ throws Exception {
+ HTable table = null;
+ try {
+ table = TEST_UTIL.createTable(tableName, fam);
+ int i = 1;
+ List puts = new ArrayList();
+ for (String labelExp : labelExps) {
+ Put put = new Put(Bytes.toBytes("row" + i));
+ put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value);
+ put.setCellVisibility(new CellVisibility(labelExp));
+ puts.add(put);
+ table.put(put);
+ i++;
+ }
+ // table.put(puts);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ return table;
+ }
+
+ public static HTable createTableAndWriteDataWithLabels(TableName tableName, long[] timestamp,
+ String... labelExps) throws Exception {
+ HTable table = null;
+ try {
+ table = TEST_UTIL.createTable(tableName, fam);
+ int i = 1;
+ List puts = new ArrayList();
+ for (String labelExp : labelExps) {
+ Put put = new Put(Bytes.toBytes("row" + i));
+ put.add(fam, qual, timestamp[i - 1], value);
+ put.setCellVisibility(new CellVisibility(labelExp));
+ puts.add(put);
+ table.put(put);
+ TEST_UTIL.getHBaseAdmin().flush(tableName.getNameAsString());
+ i++;
+ }
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ return table;
+ }
+
+ public static void addLabels() throws Exception {
+ PrivilegedExceptionAction action =
+ new PrivilegedExceptionAction() {
+ public VisibilityLabelsResponse run() throws Exception {
+ String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE };
+ try {
+ VisibilityClient.addLabels(conf, labels);
+ } catch (Throwable t) {
+ throw new IOException(t);
+ }
+ return null;
+ }
+ };
+ SUPERUSER.runAs(action);
+ }
+}
| | |