diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java index ce8b71a..24892e6 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java @@ -56,6 +56,7 @@ public class HFileContext implements HeapSize, Cloneable { private DataBlockEncoding encoding = DataBlockEncoding.NONE; /** Encryption algorithm and key used */ private Encryption.Context cryptoContext = Encryption.Context.NONE; + private long fileCreateTime; //Empty constructor. Go with setters public HFileContext() { @@ -76,12 +77,13 @@ public class HFileContext implements HeapSize, Cloneable { this.blocksize = context.blocksize; this.encoding = context.encoding; this.cryptoContext = context.cryptoContext; + this.fileCreateTime = context.fileCreateTime; } public HFileContext(boolean useHBaseChecksum, boolean includesMvcc, boolean includesTags, Compression.Algorithm compressAlgo, boolean compressTags, ChecksumType checksumType, int bytesPerChecksum, int blockSize, DataBlockEncoding encoding, - Encryption.Context cryptoContext) { + Encryption.Context cryptoContext, long fileCreateTime) { this.usesHBaseChecksum = useHBaseChecksum; this.includesMvcc = includesMvcc; this.includesTags = includesTags; @@ -94,6 +96,7 @@ public class HFileContext implements HeapSize, Cloneable { this.encoding = encoding; } this.cryptoContext = cryptoContext; + this.fileCreateTime = fileCreateTime; } /** @@ -141,6 +144,10 @@ public class HFileContext implements HeapSize, Cloneable { this.includesTags = includesTags; } + public void setFileCreateTime(long fileCreateTime) { + this.fileCreateTime = fileCreateTime; + } + public boolean isCompressTags() { return compressTags; } @@ -161,6 +168,10 @@ public class HFileContext implements HeapSize, Cloneable { return blocksize; } + public long getFileCreateTime() { + return fileCreateTime; + } + public DataBlockEncoding getDataBlockEncoding() { return encoding; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java index 5c5d75f..0d1e6ef 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java @@ -52,6 +52,7 @@ public class HFileContextBuilder { private DataBlockEncoding encoding = DataBlockEncoding.NONE; /** Crypto context */ private Encryption.Context cryptoContext = Encryption.Context.NONE; + private long fileCreateTime = 0; public HFileContextBuilder withHBaseCheckSum(boolean useHBaseCheckSum) { this.usesHBaseChecksum = useHBaseCheckSum; @@ -103,8 +104,14 @@ public class HFileContextBuilder { return this; } + public HFileContextBuilder withCreateTime(long fileCreateTime) { + this.fileCreateTime = fileCreateTime; + return this; + } + public HFileContext build() { return new HFileContext(usesHBaseChecksum, includesMvcc, includesTags, compression, - compressTags, checksumType, bytesPerChecksum, blocksize, encoding, cryptoContext); + compressTags, checksumType, bytesPerChecksum, blocksize, encoding, cryptoContext, + fileCreateTime); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java index 2bef680..d57e618 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java @@ -148,6 +148,8 @@ public abstract class AbstractHFileWriter implements HFile.Writer { int avgValueLen = entryCount == 0 ? 0 : (int) (totalValueLength / entryCount); fileInfo.append(FileInfo.AVG_VALUE_LEN, Bytes.toBytes(avgValueLen), false); + + fileInfo.append(FileInfo.CREATE_TIME_TS, Bytes.toBytes(hFileContext.getFileCreateTime()), false); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 1e97f63..ad62d71 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -541,6 +541,7 @@ public class HFile { static final byte [] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY"); static final byte [] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN"); static final byte [] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN"); + static final byte [] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS"); static final byte [] COMPARATOR = Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR"); static final byte [] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED"); public static final byte [] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java index d58ca10..f33fca4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java @@ -157,6 +157,7 @@ public class HFileReaderV2 extends AbstractHFileReader { // File info fileInfo = new FileInfo(); fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream()); + this.hfileContext.setFileCreateTime(Bytes.toLong(fileInfo.get(FileInfo.CREATE_TIME_TS))); lastKey = fileInfo.get(FileInfo.LASTKEY); avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN)); avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 6cf2ce3..1865176 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1496,6 +1496,21 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver { // return Collections.min(lastStoreFlushTimeMap.values()); } + /** + * This can be used to determine the last all file of this region were major compacted. + * + * @return the timestamp of the oldest HFile for all stores of this region + */ + public long getOldestStoreFileTime() { + long result = 0; + for (Store store : getStores().values()) { + for (StoreFile file : store.getStorefiles()) { + result = Math.min(result, file.getReader().getHFileReader().getFileContext().getFileCreateTime()); + } + } + return result; + } + ////////////////////////////////////////////////////////////////////////////// // HRegion maintenance. // diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 4669f8f..977ffe8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1344,9 +1344,13 @@ public class HRegionServer extends HasThread implements int totalStaticBloomSizeKB = 0; long totalCompactingKVs = 0; long currentCompactedKVs = 0; + long oldestHFileTs = 0; synchronized (r.stores) { stores += r.stores.size(); for (Store store : r.stores.values()) { + for (StoreFile file : store.getStorefiles()) { + oldestHFileTs = Math.min(oldestHFileTs, file.getReader().getHFileReader().getFileContext().getFileCreateTime()); + } storefiles += store.getStorefilesCount(); storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index b674fea..c432350 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -985,6 +985,7 @@ public class HStore implements Store { .withHBaseCheckSum(true) .withDataBlockEncoding(family.getDataBlockEncoding()) .withEncryptionContext(cryptoContext) + .withCreateTime(EnvironmentEdgeManager.currentTime()) .build(); return hFileContext; }