From 8158c2c438465c952bd9eff226cef11f58bcdae6 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Mon, 18 Aug 2014 17:23:20 -0700 Subject: [PATCH 2/2] HBASE-11764 Support per cell TTLs --- .../main/java/org/apache/hadoop/hbase/TagType.java | 1 + .../regionserver/GetClosestRowBeforeTracker.java | 10 ++++---- .../apache/hadoop/hbase/regionserver/HStore.java | 27 ++++++++++++++++++++-- .../hbase/regionserver/ScanQueryMatcher.java | 10 +++++++- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java index 68018c4..bf16348 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java @@ -28,4 +28,5 @@ public final class TagType { public static final byte VISIBILITY_TAG_TYPE = (byte) 2; public static final byte LOG_REPLAY_TAG_TYPE = (byte) 3; public static final byte VISIBILITY_EXP_SERIALIZATION_TAG_TYPE = (byte)4; + public static final byte TTL_TAG_TYPE = (byte)5; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java index ec676fa..bc9c1f6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hbase.util.Bytes; /** * State and utility processing {@link HRegion#getClosestRowBefore(byte[], byte[])}. - * Like {@link ScanDeleteTracker} and {@link ScanDeleteTracker} but does not + * Like {@link ScanQueryMatcher} and {@link ScanDeleteTracker} but does not * implement the {@link DeleteTracker} interface since state spans rows (There * is no update nor reset method). */ @@ -81,11 +81,11 @@ class GetClosestRowBeforeTracker { } /** - * @param kv - * @return True if this kv is expired. + * @param cell + * @return True if this cell is expired. */ - boolean isExpired(final Cell kv) { - return HStore.isExpired(kv, this.oldestts); + boolean isExpired(final Cell cell) { + return HStore.isExpired(cell, this.oldestts); } /* diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index dc593ac..e0200be 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -57,6 +57,8 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Cipher; @@ -1663,8 +1665,29 @@ public class HStore implements Store { return wantedVersions > maxVersions ? maxVersions: wantedVersions; } - static boolean isExpired(final Cell key, final long oldestTimestamp) { - return key.getTimestamp() < oldestTimestamp; + static boolean isExpired(final Cell cell, final long oldestTimestamp) { + // Do not create an Iterator or Tag objects unless the cell actually has tags + if (cell.getTagsLength() > 0) { + // Look for a TTL tag first. Use it instead of the family setting if found. + // If a cell has multiple TTLs, resolve the conflict by using the first tag + // encountered. + Iterator i = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), + cell.getTagsLength()); + while (i.hasNext()) { + Tag t = i.next(); + if (TagType.TTL_TAG_TYPE == t.getType()) { + if (t.getTagLength() == Bytes.SIZEOF_LONG) { + // Unlike in schema cell TTLs are stored in milliseconds, no need to convert + long ttl = Bytes.toLong(t.getBuffer(), t.getTagOffset(), t.getTagLength()); + return cell.getTimestamp() < ttl; + } else { + LOG.warn("TTL tag for cell " + cell + " has wrong size: have=" + t.getTagLength() + + ", want=" + Bytes.SIZEOF_LONG); + } + } + } + } + return cell.getTimestamp() < oldestTimestamp; } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java index 165eb6b..c33f3e1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java @@ -362,7 +362,7 @@ public class ScanQueryMatcher { // delete marker are not subject to other delete markers } else { // If the cell is expired and we have enough versions, skip - if (columns.hasMinVersions() && HStore.isExpired(cell, oldestUnexpiredTS)) { + if (columns.hasMinVersions() && isExpired(cell)) { return columns.getNextRowOrNextColumn(cell.getQualifierArray(), qualifierOffset, qualifierLength); } @@ -452,6 +452,14 @@ public class ScanQueryMatcher { return colChecker; } + /** + * @param cell + * @return True if this cell is expired. + */ + boolean isExpired(final Cell cell) { + return HStore.isExpired(cell, this.oldestUnexpiredTS); + } + /** Handle partial-drop-deletes. As we match keys in order, when we have a range from which * we can drop deletes, we can set retainDeletesInOutput to false for the duration of this * range only, and maintain consistency. */ -- 1.8.5.2 (Apple Git-48)