From 3f3bbe24e9418a0d86cc68b6044cf80f4176b1c8 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 6 Dec 2017 22:44:52 -0800 Subject: [PATCH] HBASE-19134 Make WALKey an Interface; expose Read-Only version to CPs --- .../hbase/test/IntegrationTestBigLinkedList.java | 1 + .../hbase/test/IntegrationTestLoadAndVerify.java | 5 +- .../hadoop/hbase/mapreduce/WALInputFormat.java | 6 +- .../apache/hadoop/hbase/mapreduce/WALPlayer.java | 3 +- .../hadoop/hbase/mapreduce/TestWALPlayer.java | 2 +- .../hbase/mapreduce/TestWALRecordReader.java | 3 +- .../replication/TestReplicationSmallTests.java | 8 +- .../hadoop/hbase/coprocessor/RegionObserver.java | 6 +- .../hbase/protobuf/ReplicationProtbufUtil.java | 13 +- .../apache/hadoop/hbase/regionserver/HRegion.java | 25 +- .../hadoop/hbase/regionserver/RSRpcServices.java | 1 + .../hbase/regionserver/RegionCoprocessorHost.java | 2 +- .../hadoop/hbase/regionserver/SequenceId.java | 9 +- .../hbase/regionserver/wal/AbstractFSWAL.java | 3 +- .../hadoop/hbase/regionserver/wal/AsyncFSWAL.java | 8 +- .../regionserver/wal/AsyncProtobufLogWriter.java | 4 +- .../hbase/regionserver/wal/CompressionContext.java | 2 +- .../hadoop/hbase/regionserver/wal/FSHLog.java | 4 +- .../hadoop/hbase/regionserver/wal/FSWALEntry.java | 8 +- .../hadoop/hbase/regionserver/wal/MetricsWAL.java | 2 +- .../hbase/regionserver/wal/ProtobufLogReader.java | 5 +- .../hbase/regionserver/wal/ProtobufLogWriter.java | 5 +- .../hadoop/hbase/regionserver/wal/ReaderBase.java | 4 +- .../hbase/regionserver/wal/WALActionsListener.java | 7 +- .../hadoop/hbase/regionserver/wal/WALUtil.java | 26 +- .../replication/ClusterMarkingEntryFilter.java | 4 +- .../replication/regionserver/Replication.java | 5 +- .../visibility/VisibilityReplicationEndpoint.java | 3 +- .../hadoop/hbase/wal/DisabledWALProvider.java | 2 +- .../main/java/org/apache/hadoop/hbase/wal/WAL.java | 11 +- .../java/org/apache/hadoop/hbase/wal/WALKey.java | 626 ++------------------ .../org/apache/hadoop/hbase/wal/WALKeyImpl.java | 631 +++++++++++++++++++++ .../org/apache/hadoop/hbase/wal/WALSplitter.java | 16 +- .../coprocessor/SampleRegionWALCoprocessor.java | 2 +- .../hbase/coprocessor/SimpleRegionObserver.java | 2 +- .../hbase/coprocessor/TestCoprocessorMetrics.java | 5 +- ...ObserverForAddingMutationsFromCoprocessors.java | 2 +- .../hadoop/hbase/coprocessor/TestWALObserver.java | 14 +- .../org/apache/hadoop/hbase/io/TestHeapSize.java | 2 +- .../hbase/master/TestDistributedLogSplitting.java | 4 +- .../hadoop/hbase/regionserver/TestBulkLoad.java | 10 +- .../hadoop/hbase/regionserver/TestHRegion.java | 15 +- .../regionserver/TestHRegionReplayEvents.java | 10 +- .../regionserver/TestHRegionServerBulkLoad.java | 2 +- .../hbase/regionserver/TestRecoveredEdits.java | 1 + .../hadoop/hbase/regionserver/TestWALLockup.java | 9 +- .../hbase/regionserver/wal/AbstractTestFSWAL.java | 8 +- .../regionserver/wal/AbstractTestProtobufLog.java | 4 +- .../regionserver/wal/AbstractTestWALReplay.java | 10 +- .../regionserver/wal/FaultyProtobufLogReader.java | 4 +- .../hadoop/hbase/regionserver/wal/TestFSHLog.java | 2 +- .../hbase/regionserver/wal/TestLogRollAbort.java | 4 +- .../regionserver/wal/TestLogRollingNoCluster.java | 4 +- .../regionserver/wal/TestWALActionsListener.java | 4 +- .../hbase/replication/TestReplicationSource.java | 5 +- .../TestReplicationWALEntryFilters.java | 14 +- .../TestRegionReplicaReplicationEndpoint.java | 4 +- ...stRegionReplicaReplicationEndpointNoMaster.java | 11 +- .../regionserver/TestReplicationSourceManager.java | 12 +- .../regionserver/TestWALEntryStream.java | 6 +- .../org/apache/hadoop/hbase/wal/FaultyFSLog.java | 2 +- .../hadoop/hbase/wal/TestFSHLogProvider.java | 2 +- .../org/apache/hadoop/hbase/wal/TestSecureWAL.java | 2 +- .../apache/hadoop/hbase/wal/TestWALFactory.java | 23 +- .../apache/hadoop/hbase/wal/TestWALMethods.java | 2 +- .../hadoop/hbase/wal/TestWALReaderOnSecureWAL.java | 2 +- .../apache/hadoop/hbase/wal/TestWALRootDir.java | 4 +- .../org/apache/hadoop/hbase/wal/TestWALSplit.java | 10 +- .../hadoop/hbase/wal/WALPerformanceEvaluation.java | 8 +- 69 files changed, 886 insertions(+), 799 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index b9cc69d9bf..96bd87e772 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -90,6 +90,7 @@ import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.RegionSplitter; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java index ce86fc2ed0..32c48a316e 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -60,6 +60,7 @@ import org.apache.hadoop.hbase.mapreduce.TableMapper; import org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; @@ -70,7 +71,7 @@ import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.hbase.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.mapreduce.WALPlayer; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java index 796acb925f..f7253b98ce 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -34,13 +34,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hbase.wal.WALKey; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WAL.Reader; -import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; @@ -248,7 +248,7 @@ public class WALInputFormat extends InputFormat { } /** - * handler for non-deprecated WALKey version. fold into WALRecordReader once we no longer + * handler for non-deprecated WALKeyImpl version. fold into WALRecordReader once we no longer * need to support HLogInputFormat. */ static class WALKeyRecordReader extends WALRecordReader { diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index 15c33cbe7e..294f530269 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.MapReduceCell; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java index 23b3c04a1a..bad088183c 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java index 4a8b8adff7..ede79dd9a6 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALRecordReader.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -242,7 +243,7 @@ public class TestWALRecordReader { } protected WALKey getWalKey(final long time, NavigableMap scopes) { - return new WALKey(info.getEncodedNameAsBytes(), tableName, time, mvcc, scopes); + return new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, time, mvcc, scopes); } protected WALRecordReader getReader() { diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java index 28bf2491d1..143f585dbd 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -76,7 +76,7 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.mapreduce.Job; import org.junit.Before; @@ -764,7 +764,7 @@ public class TestReplicationSmallTests extends TestReplicationBase { .setEndKey(HConstants.EMPTY_END_ROW) .build(); WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor); - Replication.scopeWALEdits(new WALKey(), edit, + Replication.scopeWALEdits(new WALKeyImpl(), edit, htable1.getConfiguration(), null); } @@ -844,7 +844,7 @@ public class TestReplicationSmallTests extends TestReplicationBase { long now = EnvironmentEdgeManager.currentTime(); edit.add(new KeyValue(rowName, famName, qualifier, now, value)); - WALKey walKey = new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes); + WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes); wal.append(hri, walKey, edit, true); wal.sync(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java index 6b5527bf2c..6e9032dd9b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -912,8 +912,6 @@ public interface RegionObserver { /** * Called before a {@link WALEdit} * replayed for this region. - * Do not amend the WALKey. It is InterfaceAudience.Private. Changing the WALKey will cause - * damage. * @param ctx the environment provided by the region server * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced * with something that doesn't expose IntefaceAudience.Private classes. @@ -925,8 +923,6 @@ public interface RegionObserver { /** * Called after a {@link WALEdit} * replayed for this region. - * Do not amend the WALKey. It is InterfaceAudience.Private. Changing the WALKey will cause - * damage. * @param ctx the environment provided by the region server * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced * with something that doesn't expose IntefaceAudience.Private classes. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java index af9690a659..a2b9c8cb7c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.wal.WALKey; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.io.SizedCellScanner; import org.apache.hadoop.hbase.ipc.HBaseRpcController; @@ -45,7 +46,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WAL.Entry; -import org.apache.hadoop.hbase.wal.WALKey; @InterfaceAudience.Private public class ReplicationProtbufUtil { @@ -108,7 +108,7 @@ public class ReplicationProtbufUtil { HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder(); for (Entry entry: entries) { entryBuilder.clear(); - // TODO: this duplicates a lot in WALKey#getBuilder + // TODO: this duplicates a lot in WALKeyImpl#getBuilder WALProtos.WALKey.Builder keyBuilder = entryBuilder.getKeyBuilder(); WALKey key = entry.getKey(); keyBuilder.setEncodedRegionName( @@ -116,7 +116,8 @@ public class ReplicationProtbufUtil { ? key.getEncodedRegionName() : encodedRegionName)); keyBuilder.setTableName(UnsafeByteOperations.unsafeWrap(key.getTablename().getName())); - keyBuilder.setLogSequenceNumber(key.getLogSeqNum()); + long sequenceId = key.getSequenceId(); + keyBuilder.setLogSequenceNumber(sequenceId); keyBuilder.setWriteTime(key.getWriteTime()); if (key.getNonce() != HConstants.NO_NONCE) { keyBuilder.setNonce(key.getNonce()); @@ -129,8 +130,8 @@ public class ReplicationProtbufUtil { uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits()); keyBuilder.addClusterIds(uuidBuilder.build()); } - if(key.getOrigLogSeqNum() > 0) { - keyBuilder.setOrigSequenceNumber(key.getOrigLogSeqNum()); + if (sequenceId > 0) { + keyBuilder.setOrigSequenceNumber(sequenceId); } WALEdit edit = entry.getEdit(); NavigableMap scopes = key.getReplicationScopes(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 1b7416669e..df710d8a1d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -191,6 +191,7 @@ import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hbase.wal.WALSplitter.MutationReplay; import org.apache.hadoop.io.MultipleIOException; @@ -3321,7 +3322,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi @Override public long getOrigLogSeqNum() { - return WALKey.NO_SEQUENCE_ID; + return SequenceId.NO_SEQUENCE_ID; } @Override @@ -4482,16 +4483,16 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } if (firstSeqIdInLog == -1) { - firstSeqIdInLog = key.getLogSeqNum(); + firstSeqIdInLog = key.getSequenceId(); } - if (currentEditSeqId > key.getLogSeqNum()) { + if (currentEditSeqId > key.getSequenceId()) { // when this condition is true, it means we have a serious defect because we need to // maintain increasing SeqId for WAL edits per region LOG.error(getRegionInfo().getEncodedName() + " : " + "Found decreasing SeqId. PreId=" + currentEditSeqId + " key=" + key + "; edit=" + val); } else { - currentEditSeqId = key.getLogSeqNum(); + currentEditSeqId = key.getSequenceId(); } currentReplaySeqId = (key.getOrigLogSeqNum() > 0) ? key.getOrigLogSeqNum() : currentEditSeqId; @@ -4549,7 +4550,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi continue; } // Now, figure if we should skip this edit. - if (key.getLogSeqNum() <= maxSeqIdInStores.get(store.getColumnFamilyDescriptor() + if (key.getSequenceId() <= maxSeqIdInStores.get(store.getColumnFamilyDescriptor() .getName())) { skippedEdits++; continue; @@ -7502,7 +7503,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi private WriteEntry doWALAppend(WALEdit walEdit, Durability durability, List clusterIds, long now, long nonceGroup, long nonce) throws IOException { return doWALAppend(walEdit, durability, clusterIds, now, nonceGroup, nonce, - WALKey.NO_SEQUENCE_ID); + SequenceId.NO_SEQUENCE_ID); } /** @@ -7512,16 +7513,16 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi long now, long nonceGroup, long nonce, long origLogSeqNum) throws IOException { Preconditions.checkArgument(walEdit != null && !walEdit.isEmpty(), "WALEdit is null or empty!"); - Preconditions.checkArgument(!walEdit.isReplay() || origLogSeqNum != WALKey.NO_SEQUENCE_ID, + Preconditions.checkArgument(!walEdit.isReplay() || origLogSeqNum != SequenceId.NO_SEQUENCE_ID, "Invalid replay sequence Id for replay WALEdit!"); // Using default cluster id, as this can only happen in the originating cluster. // A slave cluster receives the final value (not the delta) as a Put. We use HLogKey - // here instead of WALKey directly to support legacy coprocessors. - WALKey walKey = walEdit.isReplay() ? new WALKey(this.getRegionInfo().getEncodedNameAsBytes(), - this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now, clusterIds, nonceGroup, + // here instead of WALKeyImpl directly to support legacy coprocessors. + WALKeyImpl walKey = walEdit.isReplay() ? new WALKeyImpl(this.getRegionInfo().getEncodedNameAsBytes(), + this.htableDescriptor.getTableName(), SequenceId.NO_SEQUENCE_ID, now, clusterIds, nonceGroup, nonce, mvcc) : - new WALKey(this.getRegionInfo().getEncodedNameAsBytes(), - this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now, clusterIds, + new WALKeyImpl(this.getRegionInfo().getEncodedNameAsBytes(), + this.htableDescriptor.getTableName(), SequenceId.NO_SEQUENCE_ID, now, clusterIds, nonceGroup, nonce, mvcc, this.getReplicationScope()); if (walEdit.isReplay()) { walKey.setOrigLogSeqNum(origLogSeqNum); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index f5a35a457d..770f56f32d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -126,6 +126,7 @@ import org.apache.hadoop.hbase.util.Strings; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.yetus.audience.InterfaceAudience; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index 0448451bbf..efe78e3d4d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SequenceId.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SequenceId.java index 7bfda6a09a..edb9a7f296 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SequenceId.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SequenceId.java @@ -27,5 +27,12 @@ import org.apache.yetus.audience.InterfaceAudience; */ @InterfaceAudience.Private public interface SequenceId { - public long getSequenceId() throws IOException; + /** + * Used to represent when a particular wal key doesn't know/care about the sequence ordering. + */ + long NO_SEQUENCE_ID = -1; + + default long getSequenceId() { + return NO_SEQUENCE_ID; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java index 534315e47e..63961ec7ea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider.WriterBase; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.util.StringUtils; @@ -954,7 +955,7 @@ public abstract class AbstractFSWAL implements WAL { } } - protected final long stampSequenceIdAndPublishToRingBuffer(RegionInfo hri, WALKey key, + protected final long stampSequenceIdAndPublishToRingBuffer(RegionInfo hri, WALKeyImpl key, WALEdit edits, boolean inMemstore, RingBuffer ringBuffer) throws IOException { if (this.closed) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java index 18007aa0f7..4219bb9bd3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.trace.TraceUtil; import org.apache.hadoop.hbase.wal.AsyncFSWALProvider; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider.AsyncWriter; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.ipc.RemoteException; @@ -563,8 +564,9 @@ public class AsyncFSWAL extends AbstractFSWAL { @Override public long append(RegionInfo hri, WALKey key, WALEdit edits, boolean inMemstore) throws IOException { - long txid = - stampSequenceIdAndPublishToRingBuffer(hri, key, edits, inMemstore, waitingConsumePayloads); + // TODO: This cast presumes a lot! + long txid = stampSequenceIdAndPublishToRingBuffer(hri, (WALKeyImpl)key, edits, inMemstore, + waitingConsumePayloads); if (shouldScheduleConsumer()) { consumeExecutor.execute(consumer); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java index 482500e08d..56e37a9d9f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutputHelper; import org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException; import org.apache.hadoop.hbase.wal.AsyncFSWALProvider; import org.apache.hadoop.hbase.wal.WAL.Entry; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.com.google.common.base.Throwables; @@ -113,7 +114,8 @@ public class AsyncProtobufLogWriter extends AbstractProtobufLogWriter int buffered = output.buffered(); entry.setCompressionContext(compressionContext); try { - entry.getKey().getBuilder(compressor).setFollowingKvCount(entry.getEdit().size()).build() + ((WALKeyImpl)entry.getKey()). + getBuilder(compressor).setFollowingKvCount(entry.getEdit().size()).build() .writeDelimitedTo(asyncOutputWrapper); } catch (IOException e) { throw new AssertionError("should not happen", e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java index 22ac499581..7a2adf21bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java @@ -35,7 +35,7 @@ public class CompressionContext { static final String ENABLE_WAL_TAGS_COMPRESSION = "hbase.regionserver.wal.tags.enablecompression"; - // visible only for WALKey, until we move everything into o.a.h.h.wal + // visible only for WALKeyImpl, until we move everything into o.a.h.h.wal public final Dictionary regionDict; public final Dictionary tableDict; public final Dictionary familyDict; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index 15a6a4181f..26ca947e2a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.wal.FSHLogProvider; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALPrettyPrinter; import org.apache.hadoop.hbase.wal.WALProvider.Writer; import org.apache.hadoop.hbase.wal.WALSplitter; @@ -443,7 +444,8 @@ public class FSHLog extends AbstractFSWAL { @Override public long append(final RegionInfo hri, final WALKey key, final WALEdit edits, final boolean inMemstore) throws IOException { - return stampSequenceIdAndPublishToRingBuffer(hri, key, edits, inMemstore, + // TODO: Cast presumes a bunch. + return stampSequenceIdAndPublishToRingBuffer(hri, (WALKeyImpl)key, edits, inMemstore, disruptor.getRingBuffer()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java index a928ad580e..14a0776db1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -35,7 +35,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CollectionUtils; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; @@ -57,7 +57,7 @@ class FSWALEntry extends Entry { private final transient RegionInfo regionInfo; private final transient Set familyNames; - FSWALEntry(final long txid, final WALKey key, final WALEdit edit, + FSWALEntry(final long txid, final WALKeyImpl key, final WALEdit edit, final RegionInfo regionInfo, final boolean inMemstore) { super(key, edit); this.inMemstore = inMemstore; @@ -116,7 +116,7 @@ class FSWALEntry extends Entry { PrivateCellUtil.setSequenceId(c, regionSequenceId); } } - getKey().setWriteEntry(we); + this.key.setWriteEntry(we); return regionSequenceId; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java index ba9c0e6a69..f006866715 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java @@ -25,9 +25,9 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.wal.WALKey; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.util.StringUtils; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java index 78c055e1be..57db832325 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java @@ -28,6 +28,7 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -364,10 +365,10 @@ public class ProtobufLogReader extends ReaderBase { "probably an unexpected EOF, ignoring. current offset=" + this.inputStream.getPos()); } WALKey walKey = builder.build(); - entry.getKey().readFieldsFromPb(walKey, this.byteStringUncompressor); + ((WALKeyImpl)entry.getKey()).readFieldsFromPb(walKey, this.byteStringUncompressor); if (!walKey.hasFollowingKvCount() || 0 == walKey.getFollowingKvCount()) { if (LOG.isTraceEnabled()) { - LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos()); + LOG.trace("WALKeyImpl has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos()); } continue; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java index d1e72f764c..64acfba340 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALHeader; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALTrailer; @@ -50,8 +51,8 @@ public class ProtobufLogWriter extends AbstractProtobufLogWriter @Override public void append(Entry entry) throws IOException { entry.setCompressionContext(compressionContext); - entry.getKey().getBuilder(compressor).setFollowingKvCount(entry.getEdit().size()).build() - .writeDelimitedTo(output); + ((WALKeyImpl)entry.getKey()).getBuilder(compressor). + setFollowingKvCount(entry.getEdit().size()).build().writeDelimitedTo(output); for (Cell cell : entry.getEdit().getCells()) { // cellEncoder must assume little about the stream, since we write PB and cells in turn. cellEncoder.write(cell); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java index a30c29cd64..431a5dec44 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; @InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.PHOENIX}) public abstract class ReaderBase implements AbstractFSWALProvider.Reader { @@ -92,7 +92,7 @@ public abstract class ReaderBase implements AbstractFSWALProvider.Reader { public Entry next(Entry reuse) throws IOException { Entry e = reuse; if (e == null) { - e = new Entry(new WALKey(), new WALEdit()); + e = new Entry(new WALKeyImpl(), new WALEdit()); } if (compressionContext != null) { e.setCompressionContext(compressionContext); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java index a25b19d384..13ffac7562 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java @@ -74,13 +74,8 @@ public interface WALActionsListener { /** * Called before each write. - * @param info - * @param logKey - * @param logEdit */ - default void visitLogEntryBeforeWrite( - RegionInfo info, WALKey logKey, WALEdit logEdit - ) {} + default void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) {} /** * @param logKey diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java index 816a3b88d2..961c5a1ca7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.shaded.com.google.protobuf.TextFormat; @@ -59,11 +59,11 @@ public class WALUtil { *

This write is for internal use only. Not for external client consumption. * @param mvcc Used by WAL to get sequence Id for the waledit. */ - public static WALKey writeCompactionMarker(WAL wal, + public static WALKeyImpl writeCompactionMarker(WAL wal, NavigableMap replicationScope, RegionInfo hri, final CompactionDescriptor c, MultiVersionConcurrencyControl mvcc) throws IOException { - WALKey walKey = writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), mvcc); + WALKeyImpl walKey = writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), mvcc); if (LOG.isTraceEnabled()) { LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c)); } @@ -75,10 +75,10 @@ public class WALUtil { * *

This write is for internal use only. Not for external client consumption. */ - public static WALKey writeFlushMarker(WAL wal, NavigableMap replicationScope, + public static WALKeyImpl writeFlushMarker(WAL wal, NavigableMap replicationScope, RegionInfo hri, final FlushDescriptor f, boolean sync, MultiVersionConcurrencyControl mvcc) throws IOException { - WALKey walKey = doFullAppendTransaction(wal, replicationScope, hri, + WALKeyImpl walKey = doFullAppendTransaction(wal, replicationScope, hri, WALEdit.createFlushWALEdit(hri, f), mvcc, sync); if (LOG.isTraceEnabled()) { LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f)); @@ -90,11 +90,11 @@ public class WALUtil { * Write a region open marker indicating that the region is opened. * This write is for internal use only. Not for external client consumption. */ - public static WALKey writeRegionEventMarker(WAL wal, + public static WALKeyImpl writeRegionEventMarker(WAL wal, NavigableMap replicationScope, RegionInfo hri, final RegionEventDescriptor r, final MultiVersionConcurrencyControl mvcc) throws IOException { - WALKey walKey = writeMarker(wal, replicationScope, hri, + WALKeyImpl walKey = writeMarker(wal, replicationScope, hri, WALEdit.createRegionEventWALEdit(hri, r), mvcc); if (LOG.isTraceEnabled()) { LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r)); @@ -112,11 +112,11 @@ public class WALUtil { * @return walKey with sequenceid filled out for this bulk load marker * @throws IOException We will throw an IOException if we can not append to the HLog. */ - public static WALKey writeBulkLoadMarkerAndSync(final WAL wal, + public static WALKeyImpl writeBulkLoadMarkerAndSync(final WAL wal, final NavigableMap replicationScope, final RegionInfo hri, final WALProtos.BulkLoadDescriptor desc, final MultiVersionConcurrencyControl mvcc) throws IOException { - WALKey walKey = writeMarker(wal, replicationScope, hri, WALEdit.createBulkLoadEvent(hri, desc), + WALKeyImpl walKey = writeMarker(wal, replicationScope, hri, WALEdit.createBulkLoadEvent(hri, desc), mvcc); if (LOG.isTraceEnabled()) { LOG.trace("Appended Bulk Load marker " + TextFormat.shortDebugString(desc)); @@ -124,7 +124,7 @@ public class WALUtil { return walKey; } - private static WALKey writeMarker(final WAL wal, + private static WALKeyImpl writeMarker(final WAL wal, final NavigableMap replicationScope, final RegionInfo hri, final WALEdit edit, final MultiVersionConcurrencyControl mvcc) throws IOException { @@ -138,14 +138,14 @@ public class WALUtil { * Good for case of adding a single edit or marker to the WAL. * *

This write is for internal use only. Not for external client consumption. - * @return WALKey that was added to the WAL. + * @return WALKeyImpl that was added to the WAL. */ - public static WALKey doFullAppendTransaction(final WAL wal, + public static WALKeyImpl doFullAppendTransaction(final WAL wal, final NavigableMap replicationScope, final RegionInfo hri, final WALEdit edit, final MultiVersionConcurrencyControl mvcc, final boolean sync) throws IOException { // TODO: Pass in current time to use? - WALKey walKey = new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), + WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), System.currentTimeMillis(), mvcc, replicationScope); long trx = MultiVersionConcurrencyControl.NONE; try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java index c629e16d38..6dc50019c8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.apache.hadoop.hbase.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WAL.Entry; @@ -55,7 +55,7 @@ public class ClusterMarkingEntryFilter implements WALEntryFilter { if (replicationEndpoint.canReplicateToSameCluster() || !entry.getKey().getClusterIds().contains(peerClusterId)) { WALEdit edit = entry.getEdit(); - WALKey logKey = entry.getKey(); + WALKeyImpl logKey = (WALKeyImpl)entry.getKey(); if (edit != null && !edit.isEmpty()) { // Mark that the current cluster has the change diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java index 0e4aadd673..2a2df60303 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java @@ -31,6 +31,8 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; @@ -58,7 +60,6 @@ import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner; import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner; -import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.zookeeper.KeeperException; @@ -266,7 +267,7 @@ public class Replication implements } } if ((!replicationForBulkLoadEnabled && !foundOtherEdits) || logEdit.isReplay()) { - logKey.serializeReplicationScope(false); + ((WALKeyImpl)logKey).serializeReplicationScope(false); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityReplicationEndpoint.java index 4c86323ef6..038b79973e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityReplicationEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityReplicationEndpoint.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.replication.ReplicationEndpoint; @@ -111,7 +112,7 @@ public class VisibilityReplicationEndpoint implements ReplicationEndpoint { newEdit.add(cell); } } - newEntries.add(new Entry(entry.getKey(), newEdit)); + newEntries.add(new Entry(((WALKeyImpl)entry.getKey()), newEdit)); } replicateContext.setEntries(newEntries); return delegator.replicate(replicateContext); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java index 76b65a76eb..b1d81ee30d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java index e319255951..472423841e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -230,10 +230,11 @@ public interface WAL extends Closeable, WALFileLengthProvider { */ class Entry { private final WALEdit edit; - private final WALKey key; + // protected so subclasses can get at WALKeyImpl rather than WALKey. + protected final WALKeyImpl key; public Entry() { - this(new WALKey(), new WALEdit()); + this(new WALKeyImpl(), new WALEdit()); } /** @@ -242,7 +243,7 @@ public interface WAL extends Closeable, WALFileLengthProvider { * @param edit log's edit * @param key log's key */ - public Entry(WALKey key, WALEdit edit) { + public Entry(WALKeyImpl key, WALEdit edit) { this.key = key; this.edit = edit; } @@ -272,7 +273,7 @@ public interface WAL extends Closeable, WALFileLengthProvider { * Compression context */ public void setCompressionContext(CompressionContext compressionContext) { - key.setCompressionContext(compressionContext); + ((WALKeyImpl)key).setCompressionContext(compressionContext); } public boolean hasSerialReplicationScope () { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java index fd40ec4dda..476e853a3f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java @@ -1,647 +1,85 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.hadoop.hbase.wal; -import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.SequenceId; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.NonceKey; +import sun.security.krb5.internal.crypto.Nonce; -import java.io.IOException; -import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; -import java.util.TreeMap; import java.util.UUID; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; -import org.apache.hadoop.hbase.regionserver.SequenceId; -// imports for things that haven't moved from regionserver.wal yet. -import org.apache.hadoop.hbase.regionserver.wal.CompressionContext; -import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FamilyScope; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.ScopeType; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - -/** - * A Key for an entry in the WAL. - * - * The log intermingles edits to many tables and rows, so each log entry - * identifies the appropriate table and row. Within a table and row, they're - * also sorted. - * - *

Some Transactional edits (START, COMMIT, ABORT) will not have an associated row. - * - */ -// TODO: Key and WALEdit are never used separately, or in one-to-many relation, for practical -// purposes. They need to be merged into WALEntry. -@InterfaceAudience.Private -public class WALKey implements SequenceId, Comparable { - - /** - * Used to represent when a particular wal key doesn't know/care about the sequence ordering. - */ - public static final long NO_SEQUENCE_ID = -1; - - @InterfaceAudience.Private // For internal use only. - public MultiVersionConcurrencyControl getMvcc() { - return mvcc; - } - - /** - * Use it to complete mvcc transaction. This WALKey was part of - * (the transaction is started when you call append; see the comment on FSHLog#append). To - * complete call - * {@link MultiVersionConcurrencyControl#complete(MultiVersionConcurrencyControl.WriteEntry)} - * or {@link MultiVersionConcurrencyControl#complete(MultiVersionConcurrencyControl.WriteEntry)} - * @return A WriteEntry gotten from local WAL subsystem. - * @see #setWriteEntry(MultiVersionConcurrencyControl.WriteEntry) - */ - @InterfaceAudience.Private // For internal use only. - public MultiVersionConcurrencyControl.WriteEntry getWriteEntry() throws InterruptedIOException { - assert this.writeEntry != null; - return this.writeEntry; - } - - @InterfaceAudience.Private // For internal use only. - public void setWriteEntry(MultiVersionConcurrencyControl.WriteEntry writeEntry) { - assert this.writeEntry == null; - this.writeEntry = writeEntry; - // Set our sequenceid now using WriteEntry. - this.sequenceId = writeEntry.getWriteNumber(); - } - - private byte [] encodedRegionName; - - private TableName tablename; - /** - * SequenceId for this edit. Set post-construction at write-to-WAL time. Until then it is - * NO_SEQUENCE_ID. Change it so multiple threads can read it -- e.g. access is synchronized. - */ - private long sequenceId; - - /** - * Used during WAL replay; the sequenceId of the edit when it came into the system. - */ - private long origLogSeqNum = 0; - - /** Time at which this edit was written. */ - private long writeTime; - - /** The first element in the list is the cluster id on which the change has originated */ - private List clusterIds; - - private NavigableMap replicationScope; +public interface WALKey extends SequenceId, Comparable { - private long nonceGroup = HConstants.NO_NONCE; - private long nonce = HConstants.NO_NONCE; - private MultiVersionConcurrencyControl mvcc; /** - * Set in a way visible to multiple threads; e.g. synchronized getter/setters. + * Unmodifiable empty list of UUIDs. */ - private MultiVersionConcurrencyControl.WriteEntry writeEntry; - public static final List EMPTY_UUIDS = Collections.unmodifiableList(new ArrayList()); + List EMPTY_UUIDS = Collections.unmodifiableList(new ArrayList()); - private CompressionContext compressionContext; - - public WALKey() { - init(null, null, 0L, HConstants.LATEST_TIMESTAMP, - new ArrayList<>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, null); - } - - public WALKey(final NavigableMap replicationScope) { - init(null, null, 0L, HConstants.LATEST_TIMESTAMP, - new ArrayList<>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, replicationScope); - } - - @VisibleForTesting - public WALKey(final byte[] encodedRegionName, final TableName tablename, - long logSeqNum, - final long now, UUID clusterId) { - List clusterIds = new ArrayList<>(1); - clusterIds.add(clusterId); - init(encodedRegionName, tablename, logSeqNum, now, clusterIds, - HConstants.NO_NONCE, HConstants.NO_NONCE, null, null); - } - - // TODO: Fix being able to pass in sequenceid. - public WALKey(final byte[] encodedRegionName, final TableName tablename, final long now) { - init(encodedRegionName, - tablename, - NO_SEQUENCE_ID, - now, - EMPTY_UUIDS, - HConstants.NO_NONCE, - HConstants.NO_NONCE, - null, null); - } - - // TODO: Fix being able to pass in sequenceid. - public WALKey(final byte[] encodedRegionName, final TableName tablename, final long now, - final NavigableMap replicationScope) { - init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, EMPTY_UUIDS, HConstants.NO_NONCE, - HConstants.NO_NONCE, null, replicationScope); - } - - public WALKey(final byte[] encodedRegionName, final TableName tablename, final long now, - MultiVersionConcurrencyControl mvcc, final NavigableMap replicationScope) { - init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, EMPTY_UUIDS, HConstants.NO_NONCE, - HConstants.NO_NONCE, mvcc, replicationScope); - } - - public WALKey(final byte[] encodedRegionName, - final TableName tablename, - final long now, - MultiVersionConcurrencyControl mvcc) { - init(encodedRegionName, - tablename, - NO_SEQUENCE_ID, - now, - EMPTY_UUIDS, - HConstants.NO_NONCE, - HConstants.NO_NONCE, - mvcc, null); + default long estimatedSerializedSizeOf() { + return 0; } /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - *

Used by log splitting and snapshots. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - name of table - * @param logSeqNum - log sequence number - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) - * @param nonceGroup the nonceGroup - * @param nonce the nonce - * @param mvcc the mvcc associate the WALKey - * @param replicationScope the non-default replication scope - * associated with the region's column families + * @return encoded region name */ - // TODO: Fix being able to pass in sequenceid. - public WALKey(final byte[] encodedRegionName, final TableName tablename, long logSeqNum, - final long now, List clusterIds, long nonceGroup, long nonce, - MultiVersionConcurrencyControl mvcc, final NavigableMap replicationScope) { - init(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc, - replicationScope); - } + byte[] getEncodedRegionName(); /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - *

Used by log splitting and snapshots. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - name of table - * @param logSeqNum - log sequence number - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) + * @return table name */ - // TODO: Fix being able to pass in sequenceid. - public WALKey(final byte[] encodedRegionName, - final TableName tablename, - long logSeqNum, - final long now, - List clusterIds, - long nonceGroup, - long nonce, - MultiVersionConcurrencyControl mvcc) { - init(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc, null); - } + TableName getTablename(); /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename the tablename - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) - * @param nonceGroup - * @param nonce - * @param mvcc mvcc control used to generate sequence numbers and control read/write points + * @return the write time */ - public WALKey(final byte[] encodedRegionName, final TableName tablename, - final long now, List clusterIds, long nonceGroup, - final long nonce, final MultiVersionConcurrencyControl mvcc) { - init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, clusterIds, nonceGroup, nonce, mvcc, - null); - } + long getWriteTime(); - /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) - * @param nonceGroup the nonceGroup - * @param nonce the nonce - * @param mvcc mvcc control used to generate sequence numbers and control read/write points - * @param replicationScope the non-default replication scope of the column families - */ - public WALKey(final byte[] encodedRegionName, final TableName tablename, - final long now, List clusterIds, long nonceGroup, - final long nonce, final MultiVersionConcurrencyControl mvcc, - NavigableMap replicationScope) { - init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, clusterIds, nonceGroup, nonce, mvcc, - replicationScope); - } + NavigableMap getReplicationScopes(); /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - * @param logSeqNum - * @param nonceGroup - * @param nonce + * @return The nonce group */ - // TODO: Fix being able to pass in sequenceid. - public WALKey(final byte[] encodedRegionName, - final TableName tablename, - long logSeqNum, - long nonceGroup, - long nonce, - final MultiVersionConcurrencyControl mvcc) { - init(encodedRegionName, - tablename, - logSeqNum, - EnvironmentEdgeManager.currentTime(), - EMPTY_UUIDS, - nonceGroup, - nonce, - mvcc, null); - } - - @InterfaceAudience.Private - protected void init(final byte[] encodedRegionName, - final TableName tablename, - long logSeqNum, - final long now, - List clusterIds, - long nonceGroup, - long nonce, - MultiVersionConcurrencyControl mvcc, - NavigableMap replicationScope) { - this.sequenceId = logSeqNum; - this.writeTime = now; - this.clusterIds = clusterIds; - this.encodedRegionName = encodedRegionName; - this.tablename = tablename; - this.nonceGroup = nonceGroup; - this.nonce = nonce; - this.mvcc = mvcc; - if (logSeqNum != NO_SEQUENCE_ID) { - setSequenceId(logSeqNum); - } - this.replicationScope = replicationScope; - } - - // For deserialization. DO NOT USE. See setWriteEntry below. - @InterfaceAudience.Private - protected void setSequenceId(long sequenceId) { - this.sequenceId = sequenceId; + default long getNonceGroup() { + return HConstants.NO_NONCE; } /** - * @param compressionContext Compression context to use + * @return The nonce */ - public void setCompressionContext(CompressionContext compressionContext) { - this.compressionContext = compressionContext; - } - - /** @return encoded region name */ - public byte [] getEncodedRegionName() { - return encodedRegionName; + default long getNonce() { + return HConstants.NO_NONCE; } - /** @return table name */ - public TableName getTablename() { - return tablename; - } + List getClusterIds(); - /** @return log sequence number - * @deprecated Use {@link #getSequenceId()} - */ - @Deprecated - public long getLogSeqNum() { - return getSequenceId(); - } + UUID getOriginatingClusterId(); /** - * Used to set original sequenceId for WALKey during WAL replay - */ - public void setOrigLogSeqNum(final long sequenceId) { - this.origLogSeqNum = sequenceId; - } - - /** - * Return a positive long if current WALKey is created from a replay edit; a replay edit is an + * Return a positive long if current WALKeyImpl is created from a replay edit; a replay edit is an * edit that came in when replaying WALs of a crashed server. * @return original sequence number of the WALEdit */ - public long getOrigLogSeqNum() { - return this.origLogSeqNum; - } - - /** - * SequenceId is only available post WAL-assign. Calls before this will get you a - * {@link #NO_SEQUENCE_ID}. See the comment on FSHLog#append and #getWriteNumber in this method - * for more on when this sequenceId comes available. - * @return long the new assigned sequence number - */ - @Override - public long getSequenceId() { - return this.sequenceId; - } - - /** - * @return the write time - */ - public long getWriteTime() { - return this.writeTime; - } - - public NavigableMap getReplicationScopes() { - return replicationScope; - } - - /** @return The nonce group */ - public long getNonceGroup() { - return nonceGroup; - } - - /** @return The nonce */ - public long getNonce() { - return nonce; - } - - private void setReplicationScope(NavigableMap replicationScope) { - this.replicationScope = replicationScope; - } - - public void serializeReplicationScope(boolean serialize) { - if (!serialize) { - setReplicationScope(null); - } - } - - /** - * Marks that the cluster with the given clusterId has consumed the change - */ - public void addClusterId(UUID clusterId) { - if (!clusterIds.contains(clusterId)) { - clusterIds.add(clusterId); - } - } - - /** - * @return the set of cluster Ids that have consumed the change - */ - public List getClusterIds() { - return clusterIds; - } - - /** - * @return the cluster id on which the change has originated. It there is no such cluster, it - * returns DEFAULT_CLUSTER_ID (cases where replication is not enabled) - */ - public UUID getOriginatingClusterId(){ - return clusterIds.isEmpty() ? HConstants.DEFAULT_CLUSTER_ID : clusterIds.get(0); - } - - @Override - public String toString() { - return tablename + "/" + Bytes.toString(encodedRegionName) + "/" + - sequenceId; - } + long getOrigLogSeqNum(); /** * Produces a string map for this key. Useful for programmatic use and - * manipulation of the data stored in an WALKey, for example, printing + * manipulation of the data stored in an WALKeyImpl, for example, printing * as JSON. * * @return a Map containing data from this key */ - public Map toStringMap() { + default Map toStringMap() { Map stringMap = new HashMap<>(); - stringMap.put("table", tablename); - stringMap.put("region", Bytes.toStringBinary(encodedRegionName)); + stringMap.put("table", getTablename()); + stringMap.put("region", Bytes.toStringBinary(getEncodedRegionName())); stringMap.put("sequence", getSequenceId()); return stringMap; } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - return compareTo((WALKey)obj) == 0; - } - - @Override - public int hashCode() { - int result = Bytes.hashCode(this.encodedRegionName); - result ^= getSequenceId(); - result ^= this.writeTime; - return result; - } - - @Override - public int compareTo(WALKey o) { - int result = Bytes.compareTo(this.encodedRegionName, o.encodedRegionName); - if (result == 0) { - long sid = getSequenceId(); - long otherSid = o.getSequenceId(); - if (sid < otherSid) { - result = -1; - } else if (sid > otherSid) { - result = 1; - } - if (result == 0) { - if (this.writeTime < o.writeTime) { - result = -1; - } else if (this.writeTime > o.writeTime) { - return 1; - } - } - } - // why isn't cluster id accounted for? - return result; - } - - /** - * Drop this instance's tablename byte array and instead - * hold a reference to the provided tablename. This is not - * meant to be a general purpose setter - it's only used - * to collapse references to conserve memory. - */ - void internTableName(TableName tablename) { - // We should not use this as a setter - only to swap - // in a new reference to the same table name. - assert tablename.equals(this.tablename); - this.tablename = tablename; - } - - /** - * Drop this instance's region name byte array and instead - * hold a reference to the provided region name. This is not - * meant to be a general purpose setter - it's only used - * to collapse references to conserve memory. - */ - void internEncodedRegionName(byte []encodedRegionName) { - // We should not use this as a setter - only to swap - // in a new reference to the same table name. - assert Bytes.equals(this.encodedRegionName, encodedRegionName); - this.encodedRegionName = encodedRegionName; - } - - public WALProtos.WALKey.Builder getBuilder( - WALCellCodec.ByteStringCompressor compressor) throws IOException { - WALProtos.WALKey.Builder builder = WALProtos.WALKey.newBuilder(); - if (compressionContext == null) { - builder.setEncodedRegionName(UnsafeByteOperations.unsafeWrap(this.encodedRegionName)); - builder.setTableName(UnsafeByteOperations.unsafeWrap(this.tablename.getName())); - } else { - builder.setEncodedRegionName(compressor.compress(this.encodedRegionName, - compressionContext.regionDict)); - builder.setTableName(compressor.compress(this.tablename.getName(), - compressionContext.tableDict)); - } - builder.setLogSequenceNumber(getSequenceId()); - builder.setWriteTime(writeTime); - if (this.origLogSeqNum > 0) { - builder.setOrigSequenceNumber(this.origLogSeqNum); - } - if (this.nonce != HConstants.NO_NONCE) { - builder.setNonce(nonce); - } - if (this.nonceGroup != HConstants.NO_NONCE) { - builder.setNonceGroup(nonceGroup); - } - HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder(); - for (UUID clusterId : clusterIds) { - uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits()); - uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits()); - builder.addClusterIds(uuidBuilder.build()); - } - if (replicationScope != null) { - for (Map.Entry e : replicationScope.entrySet()) { - ByteString family = (compressionContext == null) - ? UnsafeByteOperations.unsafeWrap(e.getKey()) - : compressor.compress(e.getKey(), compressionContext.familyDict); - builder.addScopes(FamilyScope.newBuilder() - .setFamily(family).setScopeType(ScopeType.forNumber(e.getValue()))); - } - } - return builder; - } - - public void readFieldsFromPb(WALProtos.WALKey walKey, - WALCellCodec.ByteStringUncompressor uncompressor) - throws IOException { - if (this.compressionContext != null) { - this.encodedRegionName = uncompressor.uncompress( - walKey.getEncodedRegionName(), compressionContext.regionDict); - byte[] tablenameBytes = uncompressor.uncompress( - walKey.getTableName(), compressionContext.tableDict); - this.tablename = TableName.valueOf(tablenameBytes); - } else { - this.encodedRegionName = walKey.getEncodedRegionName().toByteArray(); - this.tablename = TableName.valueOf(walKey.getTableName().toByteArray()); - } - clusterIds.clear(); - for (HBaseProtos.UUID clusterId : walKey.getClusterIdsList()) { - clusterIds.add(new UUID(clusterId.getMostSigBits(), clusterId.getLeastSigBits())); - } - if (walKey.hasNonceGroup()) { - this.nonceGroup = walKey.getNonceGroup(); - } - if (walKey.hasNonce()) { - this.nonce = walKey.getNonce(); - } - this.replicationScope = null; - if (walKey.getScopesCount() > 0) { - this.replicationScope = new TreeMap<>(Bytes.BYTES_COMPARATOR); - for (FamilyScope scope : walKey.getScopesList()) { - byte[] family = (compressionContext == null) ? scope.getFamily().toByteArray() : - uncompressor.uncompress(scope.getFamily(), compressionContext.familyDict); - this.replicationScope.put(family, scope.getScopeType().getNumber()); - } - } - setSequenceId(walKey.getLogSequenceNumber()); - this.writeTime = walKey.getWriteTime(); - if(walKey.hasOrigSequenceNumber()) { - this.origLogSeqNum = walKey.getOrigSequenceNumber(); - } - } - - public long estimatedSerializedSizeOf() { - long size = encodedRegionName != null ? encodedRegionName.length : 0; - size += tablename != null ? tablename.toBytes().length : 0; - if (clusterIds != null) { - size += 16 * clusterIds.size(); - } - if (nonceGroup != HConstants.NO_NONCE) { - size += Bytes.SIZEOF_LONG; // nonce group - } - if (nonce != HConstants.NO_NONCE) { - size += Bytes.SIZEOF_LONG; // nonce - } - if (replicationScope != null) { - for (Map.Entry scope: replicationScope.entrySet()) { - size += scope.getKey().length; - size += Bytes.SIZEOF_INT; - } - } - size += Bytes.SIZEOF_LONG; // sequence number - size += Bytes.SIZEOF_LONG; // write time - if (origLogSeqNum > 0) { - size += Bytes.SIZEOF_LONG; // original sequence number - } - return size; - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java new file mode 100644 index 0000000000..23f7771c5f --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java @@ -0,0 +1,631 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.wal; + +import org.apache.hadoop.hbase.regionserver.SequenceId; +import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.UUID; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; +// imports for things that haven't moved from regionserver.wal yet. +import org.apache.hadoop.hbase.regionserver.wal.CompressionContext; +import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FamilyScope; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.ScopeType; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +/** + * A Key for an entry in the WAL. + * + * The log intermingles edits to many tables and rows, so each log entry + * identifies the appropriate table and row. Within a table and row, they're + * also sorted. + * + *

Some Transactional edits (START, COMMIT, ABORT) will not have an associated row. + * + */ +// TODO: Key and WALEdit are never used separately, or in one-to-many relation, for practical +// purposes. They need to be merged into WALEntry. +@InterfaceAudience.Private +public class WALKeyImpl implements WALKey { + public MultiVersionConcurrencyControl getMvcc() { + return mvcc; + } + + /** + * Use it to complete mvcc transaction. This WALKeyImpl was part of + * (the transaction is started when you call append; see the comment on FSHLog#append). To + * complete call + * {@link MultiVersionConcurrencyControl#complete(MultiVersionConcurrencyControl.WriteEntry)} + * or {@link MultiVersionConcurrencyControl#complete(MultiVersionConcurrencyControl.WriteEntry)} + * @return A WriteEntry gotten from local WAL subsystem. + * @see #setWriteEntry(MultiVersionConcurrencyControl.WriteEntry) + */ + public MultiVersionConcurrencyControl.WriteEntry getWriteEntry() throws InterruptedIOException { + assert this.writeEntry != null; + return this.writeEntry; + } + + public void setWriteEntry(MultiVersionConcurrencyControl.WriteEntry writeEntry) { + assert this.writeEntry == null; + this.writeEntry = writeEntry; + // Set our sequenceid now using WriteEntry. + this.sequenceId = writeEntry.getWriteNumber(); + } + + private byte [] encodedRegionName; + + private TableName tablename; + + /** + * SequenceId for this edit. Set post-construction at write-to-WAL time. Until then it is + * NO_SEQUENCE_ID. Change it so multiple threads can read it -- e.g. access is synchronized. + */ + private long sequenceId; + + /** + * Used during WAL replay; the sequenceId of the edit when it came into the system. + */ + private long origLogSeqNum = 0; + + /** Time at which this edit was written. */ + private long writeTime; + + /** The first element in the list is the cluster id on which the change has originated */ + private List clusterIds; + + private NavigableMap replicationScope; + + private long nonceGroup = HConstants.NO_NONCE; + private long nonce = HConstants.NO_NONCE; + private MultiVersionConcurrencyControl mvcc; + + /** + * Set in a way visible to multiple threads; e.g. synchronized getter/setters. + */ + private MultiVersionConcurrencyControl.WriteEntry writeEntry; + + private CompressionContext compressionContext; + + public WALKeyImpl() { + init(null, null, 0L, HConstants.LATEST_TIMESTAMP, + new ArrayList<>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, null); + } + + public WALKeyImpl(final NavigableMap replicationScope) { + init(null, null, 0L, HConstants.LATEST_TIMESTAMP, + new ArrayList<>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, replicationScope); + } + + @VisibleForTesting + public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, + long logSeqNum, + final long now, UUID clusterId) { + List clusterIds = new ArrayList<>(1); + clusterIds.add(clusterId); + init(encodedRegionName, tablename, logSeqNum, now, clusterIds, + HConstants.NO_NONCE, HConstants.NO_NONCE, null, null); + } + + // TODO: Fix being able to pass in sequenceid. + public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, final long now) { + init(encodedRegionName, + tablename, + NO_SEQUENCE_ID, + now, + EMPTY_UUIDS, + HConstants.NO_NONCE, + HConstants.NO_NONCE, + null, null); + } + + // TODO: Fix being able to pass in sequenceid. + public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, final long now, + final NavigableMap replicationScope) { + init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, EMPTY_UUIDS, HConstants.NO_NONCE, + HConstants.NO_NONCE, null, replicationScope); + } + + public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, final long now, + MultiVersionConcurrencyControl mvcc, final NavigableMap replicationScope) { + init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, EMPTY_UUIDS, HConstants.NO_NONCE, + HConstants.NO_NONCE, mvcc, replicationScope); + } + + public WALKeyImpl(final byte[] encodedRegionName, + final TableName tablename, + final long now, + MultiVersionConcurrencyControl mvcc) { + init(encodedRegionName, + tablename, + NO_SEQUENCE_ID, + now, + EMPTY_UUIDS, + HConstants.NO_NONCE, + HConstants.NO_NONCE, + mvcc, null); + } + + /** + * Create the log key for writing to somewhere. + * We maintain the tablename mainly for debugging purposes. + * A regionName is always a sub-table object. + *

Used by log splitting and snapshots. + * + * @param encodedRegionName Encoded name of the region as returned by + * HRegionInfo#getEncodedNameAsBytes(). + * @param tablename - name of table + * @param logSeqNum - log sequence number + * @param now Time at which this edit was written. + * @param clusterIds the clusters that have consumed the change(used in Replication) + * @param nonceGroup the nonceGroup + * @param nonce the nonce + * @param mvcc the mvcc associate the WALKeyImpl + * @param replicationScope the non-default replication scope + * associated with the region's column families + */ + // TODO: Fix being able to pass in sequenceid. + public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, long logSeqNum, + final long now, List clusterIds, long nonceGroup, long nonce, + MultiVersionConcurrencyControl mvcc, final NavigableMap replicationScope) { + init(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc, + replicationScope); + } + + /** + * Create the log key for writing to somewhere. + * We maintain the tablename mainly for debugging purposes. + * A regionName is always a sub-table object. + *

Used by log splitting and snapshots. + * + * @param encodedRegionName Encoded name of the region as returned by + * HRegionInfo#getEncodedNameAsBytes(). + * @param tablename - name of table + * @param logSeqNum - log sequence number + * @param now Time at which this edit was written. + * @param clusterIds the clusters that have consumed the change(used in Replication) + */ + // TODO: Fix being able to pass in sequenceid. + public WALKeyImpl(final byte[] encodedRegionName, + final TableName tablename, + long logSeqNum, + final long now, + List clusterIds, + long nonceGroup, + long nonce, + MultiVersionConcurrencyControl mvcc) { + init(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc, null); + } + + /** + * Create the log key for writing to somewhere. + * We maintain the tablename mainly for debugging purposes. + * A regionName is always a sub-table object. + * + * @param encodedRegionName Encoded name of the region as returned by + * HRegionInfo#getEncodedNameAsBytes(). + * @param tablename the tablename + * @param now Time at which this edit was written. + * @param clusterIds the clusters that have consumed the change(used in Replication) + * @param nonceGroup + * @param nonce + * @param mvcc mvcc control used to generate sequence numbers and control read/write points + */ + public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, + final long now, List clusterIds, long nonceGroup, + final long nonce, final MultiVersionConcurrencyControl mvcc) { + init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, clusterIds, nonceGroup, nonce, mvcc, + null); + } + + /** + * Create the log key for writing to somewhere. + * We maintain the tablename mainly for debugging purposes. + * A regionName is always a sub-table object. + * + * @param encodedRegionName Encoded name of the region as returned by + * HRegionInfo#getEncodedNameAsBytes(). + * @param tablename + * @param now Time at which this edit was written. + * @param clusterIds the clusters that have consumed the change(used in Replication) + * @param nonceGroup the nonceGroup + * @param nonce the nonce + * @param mvcc mvcc control used to generate sequence numbers and control read/write points + * @param replicationScope the non-default replication scope of the column families + */ + public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, + final long now, List clusterIds, long nonceGroup, + final long nonce, final MultiVersionConcurrencyControl mvcc, + NavigableMap replicationScope) { + init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, clusterIds, nonceGroup, nonce, mvcc, + replicationScope); + } + + /** + * Create the log key for writing to somewhere. + * We maintain the tablename mainly for debugging purposes. + * A regionName is always a sub-table object. + * + * @param encodedRegionName Encoded name of the region as returned by + * HRegionInfo#getEncodedNameAsBytes(). + * @param tablename + * @param logSeqNum + * @param nonceGroup + * @param nonce + */ + // TODO: Fix being able to pass in sequenceid. + public WALKeyImpl(final byte[] encodedRegionName, + final TableName tablename, + long logSeqNum, + long nonceGroup, + long nonce, + final MultiVersionConcurrencyControl mvcc) { + init(encodedRegionName, + tablename, + logSeqNum, + EnvironmentEdgeManager.currentTime(), + EMPTY_UUIDS, + nonceGroup, + nonce, + mvcc, null); + } + + @InterfaceAudience.Private + protected void init(final byte[] encodedRegionName, + final TableName tablename, + long logSeqNum, + final long now, + List clusterIds, + long nonceGroup, + long nonce, + MultiVersionConcurrencyControl mvcc, + NavigableMap replicationScope) { + this.sequenceId = logSeqNum; + this.writeTime = now; + this.clusterIds = clusterIds; + this.encodedRegionName = encodedRegionName; + this.tablename = tablename; + this.nonceGroup = nonceGroup; + this.nonce = nonce; + this.mvcc = mvcc; + if (logSeqNum != NO_SEQUENCE_ID) { + setSequenceId(logSeqNum); + } + this.replicationScope = replicationScope; + } + + // For deserialization. DO NOT USE. See setWriteEntry below. + @InterfaceAudience.Private + protected void setSequenceId(long sequenceId) { + this.sequenceId = sequenceId; + } + + /** + * @param compressionContext Compression context to use + */ + public void setCompressionContext(CompressionContext compressionContext) { + this.compressionContext = compressionContext; + } + + /** @return encoded region name */ + @Override + public byte [] getEncodedRegionName() { + return encodedRegionName; + } + + /** @return table name */ + @Override + public TableName getTablename() { + return tablename; + } + + /** @return log sequence number + * @deprecated Use {@link #getSequenceId()} + */ + @Deprecated + public long getLogSeqNum() { + return getSequenceId(); + } + + /** + * Used to set original sequenceId for WALKeyImpl during WAL replay + */ + public void setOrigLogSeqNum(final long sequenceId) { + this.origLogSeqNum = sequenceId; + } + + /** + * Return a positive long if current WALKeyImpl is created from a replay edit; a replay edit is an + * edit that came in when replaying WALs of a crashed server. + * @return original sequence number of the WALEdit + */ + @Override + public long getOrigLogSeqNum() { + return this.origLogSeqNum; + } + + /** + * SequenceId is only available post WAL-assign. Calls before this will get you a + * {@link SequenceId#NO_SEQUENCE_ID}. See the comment on FSHLog#append and #getWriteNumber in this + * method for more on when this sequenceId comes available. + * @return long the new assigned sequence number + */ + @Override + public long getSequenceId() { + return this.sequenceId; + } + + /** + * @return the write time + */ + @Override + public long getWriteTime() { + return this.writeTime; + } + + @Override + public NavigableMap getReplicationScopes() { + return replicationScope; + } + + /** @return The nonce group */ + @Override + public long getNonceGroup() { + return nonceGroup; + } + + /** @return The nonce */ + @Override + public long getNonce() { + return nonce; + } + + private void setReplicationScope(NavigableMap replicationScope) { + this.replicationScope = replicationScope; + } + + public void serializeReplicationScope(boolean serialize) { + if (!serialize) { + setReplicationScope(null); + } + } + + /** + * Marks that the cluster with the given clusterId has consumed the change + */ + public void addClusterId(UUID clusterId) { + if (!clusterIds.contains(clusterId)) { + clusterIds.add(clusterId); + } + } + + /** + * @return the set of cluster Ids that have consumed the change + */ + @Override + public List getClusterIds() { + return clusterIds; + } + + /** + * @return the cluster id on which the change has originated. It there is no such cluster, it + * returns DEFAULT_CLUSTER_ID (cases where replication is not enabled) + */ + @Override + public UUID getOriginatingClusterId(){ + return clusterIds.isEmpty() ? HConstants.DEFAULT_CLUSTER_ID : clusterIds.get(0); + } + + @Override + public String toString() { + return tablename + "/" + Bytes.toString(encodedRegionName) + "/" + sequenceId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + return compareTo((WALKey)obj) == 0; + } + + @Override + public int hashCode() { + int result = Bytes.hashCode(this.encodedRegionName); + result ^= getSequenceId(); + result ^= this.writeTime; + return result; + } + + @Override + public int compareTo(WALKey o) { + int result = Bytes.compareTo(this.encodedRegionName, o.getEncodedRegionName()); + if (result == 0) { + long sid = getSequenceId(); + long otherSid = o.getSequenceId(); + if (sid < otherSid) { + result = -1; + } else if (sid > otherSid) { + result = 1; + } + if (result == 0) { + if (this.writeTime < o.getWriteTime()) { + result = -1; + } else if (this.writeTime > o.getWriteTime()) { + return 1; + } + } + } + // why isn't cluster id accounted for? + return result; + } + + /** + * Drop this instance's tablename byte array and instead + * hold a reference to the provided tablename. This is not + * meant to be a general purpose setter - it's only used + * to collapse references to conserve memory. + */ + void internTableName(TableName tablename) { + // We should not use this as a setter - only to swap + // in a new reference to the same table name. + assert tablename.equals(this.tablename); + this.tablename = tablename; + } + + /** + * Drop this instance's region name byte array and instead + * hold a reference to the provided region name. This is not + * meant to be a general purpose setter - it's only used + * to collapse references to conserve memory. + */ + void internEncodedRegionName(byte []encodedRegionName) { + // We should not use this as a setter - only to swap + // in a new reference to the same table name. + assert Bytes.equals(this.encodedRegionName, encodedRegionName); + this.encodedRegionName = encodedRegionName; + } + + public WALProtos.WALKey.Builder getBuilder( + WALCellCodec.ByteStringCompressor compressor) throws IOException { + WALProtos.WALKey.Builder builder = WALProtos.WALKey.newBuilder(); + if (compressionContext == null) { + builder.setEncodedRegionName(UnsafeByteOperations.unsafeWrap(this.encodedRegionName)); + builder.setTableName(UnsafeByteOperations.unsafeWrap(this.tablename.getName())); + } else { + builder.setEncodedRegionName(compressor.compress(this.encodedRegionName, + compressionContext.regionDict)); + builder.setTableName(compressor.compress(this.tablename.getName(), + compressionContext.tableDict)); + } + builder.setLogSequenceNumber(getSequenceId()); + builder.setWriteTime(writeTime); + if (this.origLogSeqNum > 0) { + builder.setOrigSequenceNumber(this.origLogSeqNum); + } + if (this.nonce != HConstants.NO_NONCE) { + builder.setNonce(nonce); + } + if (this.nonceGroup != HConstants.NO_NONCE) { + builder.setNonceGroup(nonceGroup); + } + HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder(); + for (UUID clusterId : clusterIds) { + uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits()); + uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits()); + builder.addClusterIds(uuidBuilder.build()); + } + if (replicationScope != null) { + for (Map.Entry e : replicationScope.entrySet()) { + ByteString family = (compressionContext == null) + ? UnsafeByteOperations.unsafeWrap(e.getKey()) + : compressor.compress(e.getKey(), compressionContext.familyDict); + builder.addScopes(FamilyScope.newBuilder() + .setFamily(family).setScopeType(ScopeType.forNumber(e.getValue()))); + } + } + return builder; + } + + public void readFieldsFromPb(WALProtos.WALKey walKey, + WALCellCodec.ByteStringUncompressor uncompressor) + throws IOException { + if (this.compressionContext != null) { + this.encodedRegionName = uncompressor.uncompress( + walKey.getEncodedRegionName(), compressionContext.regionDict); + byte[] tablenameBytes = uncompressor.uncompress( + walKey.getTableName(), compressionContext.tableDict); + this.tablename = TableName.valueOf(tablenameBytes); + } else { + this.encodedRegionName = walKey.getEncodedRegionName().toByteArray(); + this.tablename = TableName.valueOf(walKey.getTableName().toByteArray()); + } + clusterIds.clear(); + for (HBaseProtos.UUID clusterId : walKey.getClusterIdsList()) { + clusterIds.add(new UUID(clusterId.getMostSigBits(), clusterId.getLeastSigBits())); + } + if (walKey.hasNonceGroup()) { + this.nonceGroup = walKey.getNonceGroup(); + } + if (walKey.hasNonce()) { + this.nonce = walKey.getNonce(); + } + this.replicationScope = null; + if (walKey.getScopesCount() > 0) { + this.replicationScope = new TreeMap<>(Bytes.BYTES_COMPARATOR); + for (FamilyScope scope : walKey.getScopesList()) { + byte[] family = (compressionContext == null) ? scope.getFamily().toByteArray() : + uncompressor.uncompress(scope.getFamily(), compressionContext.familyDict); + this.replicationScope.put(family, scope.getScopeType().getNumber()); + } + } + setSequenceId(walKey.getLogSequenceNumber()); + this.writeTime = walKey.getWriteTime(); + if(walKey.hasOrigSequenceNumber()) { + this.origLogSeqNum = walKey.getOrigSequenceNumber(); + } + } + + @Override + public long estimatedSerializedSizeOf() { + long size = encodedRegionName != null ? encodedRegionName.length : 0; + size += tablename != null ? tablename.toBytes().length : 0; + if (clusterIds != null) { + size += 16 * clusterIds.size(); + } + if (nonceGroup != HConstants.NO_NONCE) { + size += Bytes.SIZEOF_LONG; // nonce group + } + if (nonce != HConstants.NO_NONCE) { + size += Bytes.SIZEOF_LONG; // nonce + } + if (replicationScope != null) { + for (Map.Entry scope: replicationScope.entrySet()) { + size += scope.getKey().length; + size += Bytes.SIZEOF_INT; + } + } + size += Bytes.SIZEOF_LONG; // sequence number + size += Bytes.SIZEOF_LONG; // write time + if (origLogSeqNum > 0) { + size += Bytes.SIZEOF_LONG; // original sequence number + } + return size; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index 39063a2be6..05a1e71ec4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -957,14 +957,14 @@ public class WALSplitter { internify(entry); entryBuffer.add(entry); long incrHeap = entry.getEdit().heapSize() + - ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers + ClassSize.align(2 * ClassSize.REFERENCE) + // WALKeyImpl pointers 0; // TODO linkedlist entry heapInBuffer += incrHeap; return incrHeap; } private void internify(Entry entry) { - WALKey k = entry.getKey(); + WALKeyImpl k = (WALKeyImpl)entry.getKey(); k.internTableName(this.tableName); k.internEncodedRegionName(this.encodedRegionName); } @@ -1662,10 +1662,10 @@ public class WALSplitter { /** * This function is used to construct mutations from a WALEntry. It also - * reconstructs WALKey & WALEdit from the passed in WALEntry + * reconstructs WALKeyImpl & WALEdit from the passed in WALEntry * @param entry * @param cells - * @param logEntry pair of WALKey and WALEdit instance stores WALKey and WALEdit instances + * @param logEntry pair of WALKeyImpl and WALEdit instance stores WALKeyImpl and WALEdit instances * extracted from the passed in WALEntry. * @return list of Pair<MutationType, Mutation> to be replayed * @throws IOException @@ -1685,7 +1685,7 @@ public class WALSplitter { List mutations = new ArrayList<>(); Cell previousCell = null; Mutation m = null; - WALKey key = null; + WALKeyImpl key = null; WALEdit val = null; if (logEntry != null) val = new WALEdit(); @@ -1725,7 +1725,7 @@ public class WALSplitter { previousCell = cell; } - // reconstruct WALKey + // reconstruct WALKeyImpl if (logEntry != null) { org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALKey walKeyProto = entry.getKey(); @@ -1733,7 +1733,7 @@ public class WALSplitter { for (HBaseProtos.UUID uuid : entry.getKey().getClusterIdsList()) { clusterIds.add(new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits())); } - key = new WALKey(walKeyProto.getEncodedRegionName().toByteArray(), TableName.valueOf( + key = new WALKeyImpl(walKeyProto.getEncodedRegionName().toByteArray(), TableName.valueOf( walKeyProto.getTableName().toByteArray()), replaySeqId, walKeyProto.getWriteTime(), clusterIds, walKeyProto.getNonceGroup(), walKeyProto.getNonce(), null); logEntry.setFirst(key); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALCoprocessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALCoprocessor.java index 9f58fc4a47..0b9f1e3b19 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALCoprocessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALCoprocessor.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java index 1394dbd55d..d70e1d2c6b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorMetrics.java index fbcd1d5dc1..b2c106255d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorMetrics.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALKey; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -172,7 +173,7 @@ public class TestCoprocessorMetrics { @Override public void postWALWrite(ObserverContext ctx, - RegionInfo info, org.apache.hadoop.hbase.wal.WALKey logKey, + RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { walEditsCount.increment(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverForAddingMutationsFromCoprocessors.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverForAddingMutationsFromCoprocessors.java index 3e1621cf55..3d8287be31 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverForAddingMutationsFromCoprocessors.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverForAddingMutationsFromCoprocessors.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java index 9e140f0229..0dd2c8cd8d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java @@ -64,7 +64,7 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter; import org.junit.After; import org.junit.AfterClass; @@ -232,8 +232,8 @@ public class TestWALObserver { // it's where WAL write cp should occur. long now = EnvironmentEdgeManager.currentTime(); - // we use HLogKey here instead of WALKey directly to support legacy coprocessors. - long txid = log.append(hri, new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, + // we use HLogKey here instead of WALKeyImpl directly to support legacy coprocessors. + long txid = log.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, new MultiVersionConcurrencyControl(), scopes), edit, true); log.sync(txid); @@ -286,7 +286,7 @@ public class TestWALObserver { final long now = EnvironmentEdgeManager.currentTime(); long txid = log.append(hri, - new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes), + new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes), new WALEdit(), true); log.sync(txid); @@ -334,7 +334,7 @@ public class TestWALObserver { addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, EnvironmentEdgeManager.getDelegate(), wal, scopes, mvcc); } - wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit, + wal.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit, true); // sync to fs. wal.sync(); @@ -476,9 +476,9 @@ public class TestWALObserver { byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); - // uses WALKey instead of HLogKey on purpose. will only work for tests where we don't care + // uses WALKeyImpl instead of HLogKey on purpose. will only work for tests where we don't care // about legacy coprocessors - txid = wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, + txid = wal.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, ee.currentTime(), mvcc), edit, true); } if (-1 != txid) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java index da45fdadb7..6e9137907a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -80,7 +80,7 @@ public class TestHeapSize { private static final Log LOG = LogFactory.getLog(TestHeapSize.class); // List of classes implementing HeapSize // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry, HStoreKey - // KeyValue, LruBlockCache, Put, WALKey + // KeyValue, LruBlockCache, Put, WALKeyImpl @BeforeClass public static void beforeClass() throws Exception { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index bc68b28621..a5f965801c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -88,7 +88,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -687,7 +687,7 @@ public class TestDistributedLogSplitting { byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i)); e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value)); log.append(curRegionInfo, - new WALKey(curRegionInfo.getEncodedNameAsBytes(), fullTName, + new WALKeyImpl(curRegionInfo.getEncodedNameAsBytes(), fullTName, System.currentTimeMillis(), mvcc), e, true); if (0 == i % syncEvery) { log.sync(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java index 98bf48d699..f3c5da60e9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java @@ -60,7 +60,7 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; @@ -113,7 +113,7 @@ public class TestBulkLoad { familyName, storeFileNames)), anyBoolean())).thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { - WALKey walKey = invocation.getArgument(1); + WALKeyImpl walKey = invocation.getArgument(1); MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); if (mvcc != null) { MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); @@ -138,7 +138,7 @@ public class TestBulkLoad { any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)), anyBoolean())).thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { - WALKey walKey = invocation.getArgument(1); + WALKeyImpl walKey = invocation.getArgument(1); MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); if (mvcc != null) { MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); @@ -157,7 +157,7 @@ public class TestBulkLoad { any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)), anyBoolean())).thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { - WALKey walKey = invocation.getArgument(1); + WALKeyImpl walKey = invocation.getArgument(1); MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); if (mvcc != null) { MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); @@ -177,7 +177,7 @@ public class TestBulkLoad { any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)), anyBoolean())).thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { - WALKey walKey = invocation.getArgument(1); + WALKeyImpl walKey = invocation.getArgument(1); MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); if (mvcc != null) { MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 0f46f69aa1..9f48a691ed 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -156,6 +156,7 @@ import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; import org.apache.hadoop.hbase.wal.WALProvider.Writer; import org.apache.hadoop.hbase.wal.WALSplitter; @@ -655,7 +656,7 @@ public class TestHRegion { WALEdit edit = new WALEdit(); edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); - writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, + writer.append(new WAL.Entry(new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); @@ -706,7 +707,7 @@ public class TestHRegion { WALEdit edit = new WALEdit(); edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); - writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, + writer.append(new WAL.Entry(new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); @@ -809,7 +810,7 @@ public class TestHRegion { edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); } - writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, + writer.append(new WAL.Entry(new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); } @@ -906,7 +907,7 @@ public class TestHRegion { long time = System.nanoTime(); - writer.append(new WAL.Entry(new WALKey(regionName, tableName, 10, time, + writer.append(new WAL.Entry(new WALKeyImpl(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor))); writer.close(); @@ -4695,7 +4696,7 @@ public class TestHRegion { //verify append called or not verify(wal, expectAppend ? times(1) : never()) - .append((HRegionInfo)any(), (WALKey)any(), + .append((HRegionInfo)any(), (WALKeyImpl)any(), (WALEdit)any(), Mockito.anyBoolean()); // verify sync called or not @@ -5914,7 +5915,7 @@ public class TestHRegion { /** * Utility method to setup a WAL mock. - * Needs to do the bit where we close latch on the WALKey on append else test hangs. + * Needs to do the bit where we close latch on the WALKeyImpl on append else test hangs. * @return * @throws IOException */ @@ -5925,7 +5926,7 @@ public class TestHRegion { thenAnswer(new Answer() { @Override public Long answer(InvocationOnMock invocation) throws Throwable { - WALKey key = invocation.getArgument(1); + WALKeyImpl key = invocation.getArgument(1); MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); key.setWriteEntry(we); return 1L; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java index bab5b2673b..a96b21148d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -299,7 +299,7 @@ public class TestHRegionReplayEvents { put.setDurability(Durability.SKIP_WAL); MutationReplay mutation = new MutationReplay(MutationType.PUT, put, 0, 0); region.batchReplay(new MutationReplay[] {mutation}, - entry.getKey().getLogSeqNum()); + entry.getKey().getSequenceId()); return Integer.parseInt(Bytes.toString(put.getRow())); } @@ -1259,7 +1259,7 @@ public class TestHRegionReplayEvents { } FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0)); if (flush != null) { - secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getLogSeqNum()); + secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getSequenceId()); } } @@ -1299,7 +1299,7 @@ public class TestHRegionReplayEvents { } FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0)); if (flush != null) { - secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getLogSeqNum()); + secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getSequenceId()); } else { replayEdit(secondaryRegion, entry); } @@ -1333,7 +1333,7 @@ public class TestHRegionReplayEvents { } FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0)); if (flush != null) { - secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getLogSeqNum()); + secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getSequenceId()); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java index 723b5706df..a856bb2183 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java index 2c9a4375ed..6c269447c7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter; import org.junit.Rule; import org.junit.Test; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java index 5d082451b0..d78f76f0e2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider.Writer; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; @@ -229,7 +230,7 @@ public class TestWALLockup { // edit. WAL subsystem doesn't care. Put put = new Put(bytes); put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes); - WALKey key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), + WALKeyImpl key = new WALKeyImpl(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(), mvcc, scopes); WALEdit edit = new WALEdit(); CellScanner CellScanner = put.cellScanner(); @@ -406,7 +407,7 @@ public class TestWALLockup { try { Put put = new Put(bytes); put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes); - WALKey key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), + WALKeyImpl key = new WALKeyImpl(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(), mvcc, scopes); WALEdit edit = new WALEdit(); CellScanner CellScanner = put.cellScanner(); @@ -438,7 +439,7 @@ public class TestWALLockup { // make RingBufferEventHandler sleep 1s, so the following sync // endOfBatch=false - key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), TableName.valueOf("sleep"), + key = new WALKeyImpl(region.getRegionInfo().getEncodedNameAsBytes(), TableName.valueOf("sleep"), System.currentTimeMillis(), mvcc, scopes); dodgyWAL2.append(region.getRegionInfo(), key, edit, true); @@ -462,7 +463,7 @@ public class TestWALLockup { e1.printStackTrace(); } // make append throw DamagedWALException - key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), + key = new WALKeyImpl(region.getRegionInfo().getEncodedNameAsBytes(), TableName.valueOf("DamagedWALException"), System.currentTimeMillis(), mvcc, scopes); dodgyWAL2.append(region.getRegionInfo(), key, edit, true); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java index b736fae80d..9481018ace 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; +import org.apache.hadoop.hbase.regionserver.SequenceId; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdge; @@ -66,6 +67,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -159,8 +161,8 @@ public abstract class AbstractTestFSWAL { long timestamp = System.currentTimeMillis(); WALEdit cols = new WALEdit(); cols.add(new KeyValue(row, row, row, timestamp, row)); - WALKey key = new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(), - WALKey.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, + WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(), + SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes); log.append(hri, key, cols, true); } @@ -415,7 +417,7 @@ public abstract class AbstractTestFSWAL { goslow.set(true); for (int i = 0; i < countPerFamily; i++) { final RegionInfo info = region.getRegionInfo(); - final WALKey logkey = new WALKey(info.getEncodedNameAsBytes(), tableName, + final WALKeyImpl logkey = new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), clusterIds, -1, -1, region.getMVCC(), scopes); wal.append(info, logkey, edits, true); region.getMVCC().completeAndWait(logkey.getWriteEntry()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestProtobufLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestProtobufLog.java index 2f3e9b94f4..c0510d33b1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestProtobufLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestProtobufLog.java @@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -151,7 +151,7 @@ public abstract class AbstractTestProtobufLog { // Write log in pb format. writer = createWriter(path); for (int i = 0; i < recordCount; ++i) { - WALKey key = new WALKey( + WALKeyImpl key = new WALKeyImpl( hri.getEncodedNameAsBytes(), tableName, i, timestamp, HConstants.DEFAULT_CLUSTER_ID); WALEdit edit = new WALEdit(); for (int j = 0; j < columnCount; ++j) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java index 5acbf23625..60951aa068 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java @@ -98,7 +98,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hdfs.DFSInputStream; import org.junit.After; @@ -801,14 +801,14 @@ public abstract class AbstractTestWALReplay { long now = ee.currentTime(); edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName, now, rowName)); - wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit, + wal.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit, true); // Delete the c family to verify deletes make it over. edit = new WALEdit(); now = ee.currentTime(); edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now, KeyValue.Type.DeleteFamily)); - wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit, + wal.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes), edit, true); // Sync. @@ -1140,9 +1140,9 @@ public abstract class AbstractTestWALReplay { } } - private WALKey createWALKey(final TableName tableName, final HRegionInfo hri, + private WALKeyImpl createWALKey(final TableName tableName, final HRegionInfo hri, final MultiVersionConcurrencyControl mvcc, NavigableMap scopes) { - return new WALKey(hri.getEncodedNameAsBytes(), tableName, 999, mvcc, scopes); + return new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, 999, mvcc, scopes); } private WALEdit createWALEdit(final byte[] rowName, final byte[] family, EnvironmentEdge ee, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java index a78720dc37..cc06e98ae1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java @@ -23,7 +23,7 @@ import java.util.Queue; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; public class FaultyProtobufLogReader extends ProtobufLogReader { @@ -44,7 +44,7 @@ public class FaultyProtobufLogReader extends ProtobufLogReader { if (nextQueue.isEmpty()) { // Read the whole thing at once and fake reading boolean b; do { - Entry e = new Entry(new WALKey(), new WALEdit()); + Entry e = new Entry(new WALKeyImpl(), new WALEdit()); if (compressionContext != null) { e.setCompressionContext(compressionContext); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java index 05addd2e44..4f60daa34a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java index 1a01de5ccb..665ceeb9a9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java @@ -49,7 +49,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.After; @@ -204,7 +204,7 @@ public class TestLogRollAbort { for(byte[] fam : htd.getFamiliesKeys()) { scopes.put(fam, 0); } - log.append(regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName, + log.append(regioninfo, new WALKeyImpl(regioninfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs, true); } // Send the data to HDFS datanodes and close the HDFS writer diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java index 3ba12ed041..c990680b28 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -162,7 +162,7 @@ public class TestLogRollingNoCluster { for(byte[] fam : htd.getFamiliesKeys()) { scopes.put(fam, 0); } - final long txid = wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), + final long txid = wal.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), TableName.META_TABLE_NAME, now, mvcc, scopes), edit, true); Threads.sleep(ThreadLocalRandom.current().nextInt(5)); wal.sync(txid); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java index ec8e77123c..6696ce3fbd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -115,7 +115,7 @@ public class TestWALActionsListener { for(byte[] fam : htd.getFamiliesKeys()) { scopes.put(fam, 0); } - final long txid = wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), + final long txid = wal.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), TableName.valueOf(b), 0, mvcc, scopes), edit, true); wal.sync(txid); if (i == 10) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java index 71b4def5d2..122860532c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java @@ -34,7 +34,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; @@ -48,9 +47,9 @@ import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.replication.regionserver.Replication; import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager; import org.apache.hadoop.hbase.wal.WAL; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALProvider; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint; import org.apache.hadoop.hbase.replication.regionserver.ReplicationSource; @@ -117,7 +116,7 @@ public class TestReplicationSource { KeyValue kv = new KeyValue(b,b,b); WALEdit edit = new WALEdit(); edit.add(kv); - WALKey key = new WALKey(b, TableName.valueOf(b), 0, 0, + WALKeyImpl key = new WALKeyImpl(b, TableName.valueOf(b), 0, 0, HConstants.DEFAULT_CLUSTER_ID); writer.append(new WAL.Entry(key, edit)); writer.sync(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java index 9fda8bc2c2..83048d0b3a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WAL.Entry; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -60,21 +60,21 @@ public class TestReplicationWALEntryFilters { SystemTableWALEntryFilter filter = new SystemTableWALEntryFilter(); // meta - WALKey key1 = new WALKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), + WALKeyImpl key1 = new WALKeyImpl(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), TableName.META_TABLE_NAME, System.currentTimeMillis()); Entry metaEntry = new Entry(key1, null); assertNull(filter.filter(metaEntry)); // ns table - WALKey key2 = - new WALKey(new byte[0], TableName.NAMESPACE_TABLE_NAME, System.currentTimeMillis()); + WALKeyImpl key2 = + new WALKeyImpl(new byte[0], TableName.NAMESPACE_TABLE_NAME, System.currentTimeMillis()); Entry nsEntry = new Entry(key2, null); assertNull(filter.filter(nsEntry)); // user table - WALKey key3 = new WALKey(new byte[0], TableName.valueOf("foo"), System.currentTimeMillis()); + WALKeyImpl key3 = new WALKeyImpl(new byte[0], TableName.valueOf("foo"), System.currentTimeMillis()); Entry userEntry = new Entry(key3, null); assertEquals(userEntry, filter.filter(userEntry)); @@ -331,8 +331,8 @@ public class TestReplicationWALEntryFilters { } private Entry createEntry(TreeMap scopes, byte[]... kvs) { - WALKey key1 = - new WALKey(new byte[0], TableName.valueOf("foo"), System.currentTimeMillis(), scopes); + WALKeyImpl key1 = + new WALKeyImpl(new byte[0], TableName.valueOf("foo"), System.currentTimeMillis(), scopes); WALEdit edit1 = new WALEdit(); for (byte[] kv : kvs) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java index 1c46b57022..7e0f09024b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java @@ -48,7 +48,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.wal.WAL.Entry; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; @@ -412,7 +412,7 @@ public class TestRegionReplicaReplicationEndpoint { byte[] encodedRegionName = hrl.getRegionInfo().getEncodedNameAsBytes(); Entry entry = new Entry( - new WALKey(encodedRegionName, toBeDisabledTable, 1), + new WALKeyImpl(encodedRegionName, toBeDisabledTable, 1), new WALEdit()); HTU.getAdmin().disableTable(toBeDisabledTable); // disable the table diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpointNoMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpointNoMaster.java index 45682fca01..da4eb59c49 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpointNoMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpointNoMaster.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,7 +24,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -49,19 +48,16 @@ import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.TestRegionServerNoMaster; import org.apache.hadoop.hbase.replication.ReplicationEndpoint; import org.apache.hadoop.hbase.replication.ReplicationEndpoint.ReplicateContext; -import org.apache.hadoop.hbase.replication.ReplicationPeer; -import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; -import org.apache.hadoop.hbase.replication.WALEntryFilter; import org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint.RegionReplicaReplayCallable; import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ReplicateWALEntryResponse; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.ReplicationTests; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -165,7 +161,8 @@ public class TestRegionReplicaReplicationEndpointNoMaster { RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { // only keep primary region's edits if (logKey.getTablename().equals(tableName) && info.getReplicaId() == 0) { - entries.add(new Entry(logKey, logEdit)); + // Ugly cast! + entries.add(new Entry((WALKeyImpl)logKey, logEdit)); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java index 83dc636a39..27dc545422 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java @@ -82,7 +82,7 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -277,7 +277,7 @@ public abstract class TestReplicationSourceManager { LOG.info(i); final long txid = wal.append( hri, - new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), + new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true); wal.sync(txid); @@ -292,7 +292,7 @@ public abstract class TestReplicationSourceManager { for (int i = 0; i < 3; i++) { wal.append(hri, - new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), + new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true); } @@ -310,7 +310,7 @@ public abstract class TestReplicationSourceManager { "1", 0, false, false); wal.append(hri, - new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), + new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true); wal.sync(); @@ -428,7 +428,7 @@ public abstract class TestReplicationSourceManager { // 1. Get the bulk load wal edit event WALEdit logEdit = getBulkLoadWALEdit(scope); // 2. Create wal key - WALKey logKey = new WALKey(scope); + WALKeyImpl logKey = new WALKeyImpl(scope); // 3. Get the scopes for the key Replication.scopeWALEdits(logKey, logEdit, conf, manager); @@ -444,7 +444,7 @@ public abstract class TestReplicationSourceManager { NavigableMap scope = new TreeMap<>(Bytes.BYTES_COMPARATOR); WALEdit logEdit = getBulkLoadWALEdit(scope); // 2. Create wal key - WALKey logKey = new WALKey(scope); + WALKeyImpl logKey = new WALKeyImpl(scope); // 3. Enable bulk load hfile replication Configuration bulkLoadConf = HBaseConfiguration.create(conf); bulkLoadConf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java index 6570ab5bf8..d6d76d7029 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java @@ -56,7 +56,7 @@ import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.After; import org.junit.AfterClass; @@ -374,7 +374,7 @@ public class TestWALEntryStream { private void appendToLog(String key) throws IOException { final long txid = log.append(info, - new WALKey(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), + new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), getWALEdit(key), true); log.sync(txid); } @@ -391,7 +391,7 @@ public class TestWALEntryStream { private void appendToLogPlus(int count) throws IOException { final long txid = log.append(info, - new WALKey(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), + new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), getWALEdits(count), true); log.sync(txid); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java index 472eadd5f3..acf6198b8d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestFSHLogProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestFSHLogProvider.java index 93c5d4f106..7a2f32f2ae 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestFSHLogProvider.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestFSHLogProvider.java @@ -168,7 +168,7 @@ public class TestFSHLogProvider { */ WALKey getWalKey(final byte[] info, final TableName tableName, final long timestamp, NavigableMap scopes) { - return new WALKey(info, tableName, timestamp, mvcc, scopes); + return new WALKeyImpl(info, tableName, timestamp, mvcc, scopes); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java index 77d63a4dea..ac53ae9554 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java @@ -138,7 +138,7 @@ public class TestSecureWAL { for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value)); - wal.append(regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName, + wal.append(regioninfo, new WALKeyImpl(regioninfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs, true); } wal.sync(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java index 92c7458ffa..7c1af2539e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -197,7 +197,7 @@ public class TestWALFactory { edit.add(new KeyValue(rowName, family, qualifier, System.currentTimeMillis(), column)); LOG.info("Region " + i + ": " + edit); - WALKey walKey = new WALKey(infos[i].getEncodedNameAsBytes(), tableName, + WALKeyImpl walKey = new WALKeyImpl(infos[i].getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes); log.append(infos[i], walKey, edit, true); walKey.getWriteEntry(); @@ -266,7 +266,7 @@ public class TestWALFactory { for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName())); - wal.append(info, new WALKey(info.getEncodedNameAsBytes(), tableName, + wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs, true); } // Now call sync and try reading. Opening a Reader before you sync just @@ -285,7 +285,7 @@ public class TestWALFactory { for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName())); - wal.append(info, new WALKey(info.getEncodedNameAsBytes(), tableName, + wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs, true); } wal.sync(); @@ -307,7 +307,7 @@ public class TestWALFactory { for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), value)); - wal.append(info, new WALKey(info.getEncodedNameAsBytes(), tableName, + wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs, true); } // Now I should have written out lots of blocks. Sync then read. @@ -390,7 +390,7 @@ public class TestWALFactory { for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName())); - wal.append(regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName, + wal.append(regioninfo, new WALKeyImpl(regioninfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs, true); } @@ -526,7 +526,7 @@ public class TestWALFactory { final WAL log = wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()); final long txid = log.append(info, - new WALKey(info.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(), + new WALKeyImpl(info.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(), mvcc, scopes), cols, true); log.sync(txid); @@ -589,7 +589,7 @@ public class TestWALFactory { HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); final WAL log = wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()); final long txid = log.append(hri, - new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(), + new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(), mvcc, scopes), cols, true); log.sync(txid); @@ -646,7 +646,7 @@ public class TestWALFactory { cols.add(new KeyValue(row, Bytes.toBytes("column"), Bytes.toBytes(Integer.toString(i)), timestamp, new byte[]{(byte) (i + '0')})); - log.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, + log.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), cols, true); } log.sync(); @@ -656,7 +656,7 @@ public class TestWALFactory { cols.add(new KeyValue(row, Bytes.toBytes("column"), Bytes.toBytes(Integer.toString(11)), timestamp, new byte[]{(byte) (11 + '0')})); - log.append(hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, + log.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), cols, true); log.sync(); assertEquals(COL_COUNT, visitor.increments); @@ -677,8 +677,7 @@ public class TestWALFactory { int increments = 0; @Override - public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, - WALEdit logEdit) { + public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) { increments++; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java index 243e9451a6..a6d245698f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java @@ -183,7 +183,7 @@ public class TestWALMethods { WALEdit edit = new WALEdit(); edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val")); - WALKey key = new WALKey(TEST_REGION, TEST_TABLE, seq, now, + WALKeyImpl key = new WALKeyImpl(TEST_REGION, TEST_TABLE, seq, now, HConstants.DEFAULT_CLUSTER_ID); WAL.Entry entry = new WAL.Entry(key, edit); return entry; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java index 095b3720ba..32253106cc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java @@ -127,7 +127,7 @@ public class TestWALReaderOnSecureWAL { } else { kvs.add(kv); } - wal.append(regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName, + wal.append(regioninfo, new WALKeyImpl(regioninfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs, true); } wal.sync(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java index acdb5a2155..291f81e316 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALRootDir.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -113,7 +113,7 @@ public class TestWALRootDir { } protected WALKey getWalKey(final long time, HRegionInfo hri, final long startPoint) { - return new WALKey(hri.getEncodedNameAsBytes(), tableName, time, new MultiVersionConcurrencyControl(startPoint)); + return new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, time, new MultiVersionConcurrencyControl(startPoint)); } private List getWALFiles(FileSystem fs, Path dir) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index e2188411f7..0fc0df1e64 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -379,7 +379,7 @@ public class TestWALSplit { fs.mkdirs(regiondir); long now = System.currentTimeMillis(); Entry entry = - new Entry(new WALKey(encoded, + new Entry(new WALKeyImpl(encoded, TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), new WALEdit()); Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR, @@ -401,7 +401,7 @@ public class TestWALSplit { fs.mkdirs(regiondir); long now = System.currentTimeMillis(); Entry entry = - new Entry(new WALKey(encoded, + new Entry(new WALKeyImpl(encoded, TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), new WALEdit()); Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); @@ -1345,7 +1345,7 @@ public class TestWALSplit { .addCompactionOutput(output); WALEdit edit = WALEdit.createCompaction(hri, desc.build()); - WALKey key = new WALKey(hri.getEncodedNameAsBytes(), TABLE_NAME, 1, + WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), TABLE_NAME, 1, EnvironmentEdgeManager.currentTime(), HConstants.DEFAULT_CLUSTER_ID); w.append(new Entry(key, edit)); w.sync(); @@ -1362,7 +1362,7 @@ public class TestWALSplit { final long time = EnvironmentEdgeManager.currentTime(); KeyValue kv = new KeyValue(region.getBytes(), WALEdit.METAFAMILY, WALEdit.REGION_EVENT, time, regionOpenDesc.toByteArray()); - final WALKey walKey = new WALKey(region.getBytes(), TABLE_NAME, 1, time, + final WALKeyImpl walKey = new WALKeyImpl(region.getBytes(), TABLE_NAME, 1, time, HConstants.DEFAULT_CLUSTER_ID); w.append( new Entry(walKey, new WALEdit().add(kv))); @@ -1390,7 +1390,7 @@ public class TestWALSplit { final KeyValue cell = new KeyValue(row, family, qualifier, time, KeyValue.Type.Put, value); WALEdit edit = new WALEdit(); edit.add(cell); - return new Entry(new WALKey(region, table, seq, time, + return new Entry(new WALKeyImpl(region, table, seq, time, HConstants.DEFAULT_CLUSTER_ID), edit); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java index 32418033b9..de23d61e88 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -185,8 +185,8 @@ public final class WALPerformanceEvaluation extends Configured implements Tool { WALEdit walEdit = new WALEdit(); addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit); RegionInfo hri = region.getRegionInfo(); - final WALKey logkey = - new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes); + final WALKeyImpl logkey = + new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes); wal.append(hri, logkey, walEdit, true); if (!this.noSync) { if (++lastSync >= this.syncInterval) { @@ -430,7 +430,7 @@ public final class WALPerformanceEvaluation extends Configured implements Tool { break; } count++; - long seqid = e.getKey().getLogSeqNum(); + long seqid = e.getKey().getSequenceId(); if (sequenceIds.containsKey(Bytes.toString(e.getKey().getEncodedRegionName()))) { // sequenceIds should be increasing for every regions if (sequenceIds.get(Bytes.toString(e.getKey().getEncodedRegionName())) >= seqid) { -- 2.11.0 (Apple Git-81)