From 4d354c50735b5e7d211cbfec4e27cc934d0d08fb Mon Sep 17 00:00:00 2001 From: zhangduo Date: Tue, 22 Nov 2016 10:53:34 +0800 Subject: [PATCH] HBASE-17132 Cleanup deprecated code for WAL --- .../hbase/coprocessor/BaseRegionObserver.java | 17 +- .../hadoop/hbase/coprocessor/BaseWALObserver.java | 21 +- .../hadoop/hbase/coprocessor/RegionObserver.java | 53 +- .../hadoop/hbase/coprocessor/WALObserver.java | 60 +- .../hadoop/hbase/mapreduce/HLogInputFormat.java | 83 -- .../apache/hadoop/hbase/mapreduce/WALPlayer.java | 9 +- .../apache/hadoop/hbase/regionserver/HRegion.java | 20 +- .../hbase/regionserver/RegionCoprocessorHost.java | 100 +-- .../hadoop/hbase/regionserver/wal/FSHLog.java | 8 +- .../hadoop/hbase/regionserver/wal/HLogKey.java | 285 ------- .../hadoop/hbase/regionserver/wal/ReaderBase.java | 8 +- .../hbase/regionserver/wal/ReplayHLogKey.java | 54 -- .../regionserver/wal/SequenceFileLogReader.java | 309 -------- .../hbase/regionserver/wal/WALCoprocessorHost.java | 40 +- .../hbase/security/access/AccessControlLists.java | 54 +- .../access/HbaseObjectWritableFor96Migration.java | 844 --------------------- .../main/java/org/apache/hadoop/hbase/wal/WAL.java | 10 +- .../org/apache/hadoop/hbase/wal/WALFactory.java | 51 +- .../java/org/apache/hadoop/hbase/wal/WALKey.java | 17 +- .../org/apache/hadoop/hbase/wal/WALProvider.java | 2 +- .../org/apache/hadoop/hbase/wal/WALSplitter.java | 23 +- .../hbase/coprocessor/SampleRegionWALObserver.java | 76 +- .../hbase/coprocessor/SimpleRegionObserver.java | 39 +- .../coprocessor/TestRegionObserverInterface.java | 382 ++++------ .../hadoop/hbase/coprocessor/TestWALObserver.java | 106 +-- .../hbase/mapreduce/TestHLogRecordReader.java | 46 -- .../hbase/master/TestDistributedLogSplitting.java | 11 +- .../regionserver/TestFailedAppendAndSync.java | 2 +- .../hadoop/hbase/regionserver/TestHRegion.java | 33 +- .../hadoop/hbase/regionserver/TestWALLockup.java | 16 +- .../regionserver/wal/FaultyProtobufLogReader.java | 76 ++ .../wal/FaultySequenceFileLogReader.java | 80 -- .../regionserver/wal/SequenceFileLogWriter.java | 239 ------ .../wal/TestReadOldRootAndMetaEdits.java | 161 ---- .../TestReplicationWALEntryFilters.java | 29 +- .../access/HbaseObjectWritableFor96Migration.java | 816 ++++++++++++++++++++ .../security/access/TestTablePermissions.java | 4 +- .../wal/TestDefaultWALProviderWithHLogKey.java | 37 - .../hadoop/hbase/wal/TestFSHLogProvider.java | 2 - .../apache/hadoop/hbase/wal/TestWALFactory.java | 79 +- .../apache/hadoop/hbase/wal/TestWALMethods.java | 27 +- .../org/apache/hadoop/hbase/wal/TestWALSplit.java | 46 +- 42 files changed, 1242 insertions(+), 3133 deletions(-) delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayHLogKey.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadOldRootAndMetaEdits.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java index 3442b64..660c631 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.coprocessor; +import com.google.common.collect.ImmutableList; + import java.io.IOException; import java.util.List; import java.util.Map; @@ -58,13 +60,10 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WALKey; -import com.google.common.collect.ImmutableList; - /** * An abstract class that implements RegionObserver. * By extending it, you can create your own region observer without @@ -485,12 +484,6 @@ public class BaseRegionObserver implements RegionObserver { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { } - @Override - public void preWALRestore(ObserverContext env, HRegionInfo info, - HLogKey logKey, WALEdit logEdit) throws IOException { - preWALRestore(env, info, (WALKey)logKey, logEdit); - } - /** * Implementers should override this version of the method and leave the deprecated one as-is. */ @@ -500,12 +493,6 @@ public class BaseRegionObserver implements RegionObserver { } @Override - public void postWALRestore(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - postWALRestore(env, info, (WALKey)logKey, logEdit); - } - - @Override public void preBulkLoadHFile(final ObserverContext ctx, List> familyPaths) throws IOException { } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseWALObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseWALObserver.java index 1d0076a..8507db7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseWALObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseWALObserver.java @@ -21,14 +21,13 @@ package org.apache.hadoop.hbase.coprocessor; import java.io.IOException; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.wal.WALKey; /** * An abstract class that implements WALObserver. @@ -53,24 +52,12 @@ public class BaseWALObserver implements WALObserver { return false; } - @Override - public boolean preWALWrite(ObserverContext ctx, HRegionInfo info, - HLogKey logKey, WALEdit logEdit) throws IOException { - return preWALWrite(ctx, info, (WALKey)logKey, logEdit); - } - /** * Implementers should override this method and leave the deprecated version as-is. */ @Override public void postWALWrite(ObserverContext ctx, - HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { } - - @Override - public void postWALWrite(ObserverContext ctx, HRegionInfo info, - HLogKey logKey, WALEdit logEdit) throws IOException { - postWALWrite(ctx, info, (WALKey)logKey, logEdit); - } + HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {} @Override public void preWALRoll(ObserverContext ctx, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java index d9a509c..0d01baa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.coprocessor; +import com.google.common.collect.ImmutableList; + import java.io.IOException; import java.util.List; import java.util.Map; @@ -59,13 +61,10 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WALKey; -import com.google.common.collect.ImmutableList; - /** * Coprocessors implement this interface to observe and mediate client actions * on the region. @@ -1318,30 +1317,6 @@ public interface RegionObserver extends Coprocessor { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException; /** - * Called before a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} - * replayed for this region. - * - * This method is left in place to maintain binary compatibility with older - * {@link RegionObserver}s. If an implementation directly overrides - * {@link #preWALRestore(ObserverContext, HRegionInfo, WALKey, WALEdit)} then this version - * won't be called at all, barring problems with the Security Manager. To work correctly - * in the presence of a strict Security Manager, or in the case of an implementation that - * relies on a parent class to implement preWALRestore, you should implement this method - * as a call to the non-deprecated version. - * - * Users of this method will see all edits that can be treated as HLogKey. If there are - * edits that can't be treated as HLogKey they won't be offered to coprocessors that rely - * on this method. If a coprocessor gets skipped because of this mechanism, a log message - * at ERROR will be generated per coprocessor on the logger for {@link CoprocessorHost} once per - * classloader. - * - * @deprecated use {@link #preWALRestore(ObserverContext, HRegionInfo, WALKey, WALEdit)} - */ - @Deprecated - void preWALRestore(final ObserverContext ctx, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; - - /** * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} * replayed for this region. */ @@ -1349,30 +1324,6 @@ public interface RegionObserver extends Coprocessor { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException; /** - * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} - * replayed for this region. - * - * This method is left in place to maintain binary compatibility with older - * {@link RegionObserver}s. If an implementation directly overrides - * {@link #postWALRestore(ObserverContext, HRegionInfo, WALKey, WALEdit)} then this version - * won't be called at all, barring problems with the Security Manager. To work correctly - * in the presence of a strict Security Manager, or in the case of an implementation that - * relies on a parent class to implement preWALRestore, you should implement this method - * as a call to the non-deprecated version. - * - * Users of this method will see all edits that can be treated as HLogKey. If there are - * edits that can't be treated as HLogKey they won't be offered to coprocessors that rely - * on this method. If a coprocessor gets skipped because of this mechanism, a log message - * at ERROR will be generated per coprocessor on the logger for {@link CoprocessorHost} once per - * classloader. - * - * @deprecated use {@link #postWALRestore(ObserverContext, HRegionInfo, WALKey, WALEdit)} - */ - @Deprecated - void postWALRestore(final ObserverContext ctx, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; - - /** * Called before bulkLoadHFile. Users can create a StoreFile instance to * access the contents of a HFile. * diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java index 7fd03ce..344a764 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java @@ -19,17 +19,16 @@ package org.apache.hadoop.hbase.coprocessor; +import java.io.IOException; + import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; - -import java.io.IOException; +import org.apache.hadoop.hbase.wal.WALKey; /** * It's provided to have a way for coprocessors to observe, rewrite, @@ -59,31 +58,6 @@ public interface WALObserver extends Coprocessor { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException; /** - * Called before a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} - * is writen to WAL. - * - * This method is left in place to maintain binary compatibility with older - * {@link WALObserver}s. If an implementation directly overrides - * {@link #preWALWrite(ObserverContext, HRegionInfo, WALKey, WALEdit)} then this version - * won't be called at all, barring problems with the Security Manager. To work correctly - * in the presence of a strict Security Manager, or in the case of an implementation that - * relies on a parent class to implement preWALWrite, you should implement this method - * as a call to the non-deprecated version. - * - * Users of this method will see all edits that can be treated as HLogKey. If there are - * edits that can't be treated as HLogKey they won't be offered to coprocessors that rely - * on this method. If a coprocessor gets skipped because of this mechanism, a log message - * at ERROR will be generated per coprocessor on the logger for {@link CoprocessorHost} once per - * classloader. - * - * @return true if default behavior should be bypassed, false otherwise - * @deprecated use {@link #preWALWrite(ObserverContext, HRegionInfo, WALKey, WALEdit)} - */ - @Deprecated - boolean preWALWrite(ObserverContext ctx, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; - - /** * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} * is writen to WAL. */ @@ -91,30 +65,6 @@ public interface WALObserver extends Coprocessor { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException; /** - * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} - * is writen to WAL. - * - * This method is left in place to maintain binary compatibility with older - * {@link WALObserver}s. If an implementation directly overrides - * {@link #postWALWrite(ObserverContext, HRegionInfo, WALKey, WALEdit)} then this version - * won't be called at all, barring problems with the Security Manager. To work correctly - * in the presence of a strict Security Manager, or in the case of an implementation that - * relies on a parent class to implement preWALWrite, you should implement this method - * as a call to the non-deprecated version. - * - * Users of this method will see all edits that can be treated as HLogKey. If there are - * edits that can't be treated as HLogKey they won't be offered to coprocessors that rely - * on this method. If a coprocessor gets skipped because of this mechanism, a log message - * at ERROR will be generated per coprocessor on the logger for {@link CoprocessorHost} once per - * classloader. - * - * @deprecated use {@link #postWALWrite(ObserverContext, HRegionInfo, WALKey, WALEdit)} - */ - @Deprecated - void postWALWrite(ObserverContext ctx, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; - - /** * Called before rolling the current WAL * @param oldPath the path of the current wal that we are replacing * @param newPath the path of the wal we are going to create diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java deleted file mode 100644 index e727e15..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.mapreduce; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.regionserver.wal.WALEdit; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -/** - * Simple {@link InputFormat} for {@link org.apache.hadoop.hbase.wal.WAL} - * files. - * @deprecated use {@link WALInputFormat}. Remove in hadoop 3.0 - */ -@Deprecated -@InterfaceAudience.Public -public class HLogInputFormat extends InputFormat { - private static final Log LOG = LogFactory.getLog(HLogInputFormat.class); - public static final String START_TIME_KEY = "hlog.start.time"; - public static final String END_TIME_KEY = "hlog.end.time"; - - // Delegate to WALInputFormat for implementation. - private final WALInputFormat delegate = new WALInputFormat(); - - /** - * {@link RecordReader} that pulls out the legacy HLogKey format directly. - */ - static class HLogKeyRecordReader extends WALInputFormat.WALRecordReader { - @Override - public HLogKey getCurrentKey() throws IOException, InterruptedException { - if (!(currentEntry.getKey() instanceof HLogKey)) { - final IllegalStateException exception = new IllegalStateException( - "HLogInputFormat only works when given entries that have HLogKey for keys. This" + - " one had '" + currentEntry.getKey().getClass() + "'"); - LOG.error("The deprecated HLogInputFormat has to work with the deprecated HLogKey class, " + - " but HBase internals read the wal entry using some other class." + - " This is a bug; please file an issue or email the developer mailing list. It is " + - "likely that you would not have this problem if you updated to use WALInputFormat. " + - "You will need the following exception details when seeking help from the HBase " + - "community.", - exception); - throw exception; - } - return (HLogKey)currentEntry.getKey(); - } - } - - @Override - public List getSplits(JobContext context) throws IOException, - InterruptedException { - return delegate.getSplits(context, START_TIME_KEY, END_TIME_KEY); - } - - @Override - public RecordReader createRecordReader(InputSplit split, - TaskAttemptContext context) throws IOException, InterruptedException { - return new HLogKeyRecordReader(); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index 452714b..2c67baf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -80,8 +80,6 @@ public class WALPlayer extends Configured implements Tool { Configuration.addDeprecation("hlog.bulk.output", BULK_OUTPUT_CONF_KEY); Configuration.addDeprecation("hlog.input.tables", TABLES_KEY); Configuration.addDeprecation("hlog.input.tablesmap", TABLE_MAP_KEY); - Configuration.addDeprecation(HLogInputFormat.START_TIME_KEY, WALInputFormat.START_TIME_KEY); - Configuration.addDeprecation(HLogInputFormat.END_TIME_KEY, WALInputFormat.END_TIME_KEY); } private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; @@ -241,11 +239,10 @@ public class WALPlayer extends Configured implements Tool { * @return The newly created job. * @throws IOException When setting up the job fails. */ - public Job createSubmittableJob(String[] args) - throws IOException { + public Job createSubmittableJob(String[] args) throws IOException { Configuration conf = getConf(); - setupTime(conf, HLogInputFormat.START_TIME_KEY); - setupTime(conf, HLogInputFormat.END_TIME_KEY); + setupTime(conf, WALInputFormat.START_TIME_KEY); + setupTime(conf, WALInputFormat.END_TIME_KEY); Path inputDir = new Path(args[0]); String[] tables = args[1].split(","); String[] tableMap; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 6415672..6b41bc4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -19,6 +19,13 @@ package org.apache.hadoop.hbase.regionserver; import static org.apache.hadoop.hbase.HConstants.REPLICATION_SCOPE_LOCAL; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.io.Closeables; + import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; @@ -137,8 +144,6 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.regionserver.wal.ReplayHLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.regionserver.wal.WALUtil; import org.apache.hadoop.hbase.security.User; @@ -182,13 +187,6 @@ import org.apache.hadoop.util.StringUtils; import org.apache.htrace.Trace; import org.apache.htrace.TraceScope; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.io.Closeables; - @SuppressWarnings("deprecation") @InterfaceAudience.Private @@ -3258,7 +3256,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi long txid; if (replay) { // use wal key from the original - walKey = new ReplayHLogKey(this.getRegionInfo().getEncodedNameAsBytes(), + walKey = new WALKey(this.getRegionInfo().getEncodedNameAsBytes(), this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now, mutation.getClusterIds(), currentNonceGroup, currentNonce, mvcc); walKey.setOrigLogSeqNum(batchOp.getReplaySequenceId()); @@ -3277,7 +3275,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi writeEntry = mvcc.begin(); } // we use HLogKey here instead of WALKey directly to support legacy coprocessors. - walKey = new HLogKey(this.getRegionInfo().getEncodedNameAsBytes(), + walKey = new WALKey(this.getRegionInfo().getEncodedNameAsBytes(), this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now, mutation.getClusterIds(), currentNonceGroup, currentNonce, mvcc, this.getReplicationScope()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index f6388fa..2ddb1c0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -19,29 +19,26 @@ package org.apache.hadoop.hbase.regionserver; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.protobuf.Message; +import com.google.protobuf.Service; + import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.NavigableSet; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.regex.Matcher; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.protobuf.Message; -import com.google.protobuf.Service; - import org.apache.commons.collections.map.AbstractReferenceMap; import org.apache.commons.collections.map.ReferenceMap; import org.apache.commons.lang.ClassUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -53,6 +50,8 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; @@ -79,13 +78,12 @@ import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.regionserver.Region.Operation; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CoprocessorClassLoader; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.wal.WALKey; /** * Implements the coprocessor environment and runtime support for coprocessors @@ -114,8 +112,6 @@ public class RegionCoprocessorHost private Region region; private RegionServerServices rsServices; ConcurrentMap sharedData; - private final boolean useLegacyPre; - private final boolean useLegacyPost; /** * Constructor @@ -129,14 +125,6 @@ public class RegionCoprocessorHost this.region = region; this.rsServices = services; this.sharedData = sharedData; - // Pick which version of the WAL related events we'll call. - // This way we avoid calling the new version on older RegionObservers so - // we can maintain binary compatibility. - // See notes in javadoc for RegionObserver - useLegacyPre = useLegacyMethod(impl.getClass(), "preWALRestore", ObserverContext.class, - HRegionInfo.class, WALKey.class, WALEdit.class); - useLegacyPost = useLegacyMethod(impl.getClass(), "postWALRestore", ObserverContext.class, - HRegionInfo.class, WALKey.class, WALEdit.class); } /** @return the region */ @@ -422,31 +410,6 @@ public class RegionCoprocessorHost } /** - * HBASE-4014 : This is used by coprocessor hooks which are not declared to throw exceptions. - * - * For example, {@link - * org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost#preOpen()} and - * {@link org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost#postOpen()} are such hooks. - * - * See also - * {@link org.apache.hadoop.hbase.master.MasterCoprocessorHost#handleCoprocessorThrowable( - * CoprocessorEnvironment, Throwable)} - * @param env The coprocessor that threw the exception. - * @param e The exception that was thrown. - */ - private void handleCoprocessorThrowableNoRethrow( - final CoprocessorEnvironment env, final Throwable e) { - try { - handleCoprocessorThrowable(env,e); - } catch (IOException ioe) { - // We cannot throw exceptions from the caller hook, so ignore. - LOG.warn( - "handleCoprocessorThrowable() threw an IOException while attempting to handle Throwable " + - e + ". Ignoring.",e); - } - } - - /** * Invoked before a region open. * * @throws IOException Signals that an I/O exception has occurred. @@ -1470,33 +1433,12 @@ public class RegionCoprocessorHost @Override public void call(RegionObserver oserver, ObserverContext ctx) throws IOException { - // Once we don't need to support the legacy call, replace RegionOperation with a version - // that's ObserverContext and avoid this cast. - final RegionEnvironment env = (RegionEnvironment)ctx.getEnvironment(); - if (env.useLegacyPre) { - if (logKey instanceof HLogKey) { - oserver.preWALRestore(ctx, info, (HLogKey)logKey, logEdit); - } else { - legacyWarning(oserver.getClass(), "There are wal keys present that are not HLogKey."); - } - } else { - oserver.preWALRestore(ctx, info, logKey, logEdit); - } + oserver.preWALRestore(ctx, info, logKey, logEdit); } }); } /** - * @return true if default behavior should be bypassed, false otherwise - * @deprecated use {@link #preWALRestore(HRegionInfo, WALKey, WALEdit)}; as of 2.0, remove in 3.0 - */ - @Deprecated - public boolean preWALRestore(final HRegionInfo info, final HLogKey logKey, - final WALEdit logEdit) throws IOException { - return preWALRestore(info, (WALKey)logKey, logEdit); - } - - /** * @param info * @param logKey * @param logEdit @@ -1508,32 +1450,12 @@ public class RegionCoprocessorHost @Override public void call(RegionObserver oserver, ObserverContext ctx) throws IOException { - // Once we don't need to support the legacy call, replace RegionOperation with a version - // that's ObserverContext and avoid this cast. - final RegionEnvironment env = (RegionEnvironment)ctx.getEnvironment(); - if (env.useLegacyPost) { - if (logKey instanceof HLogKey) { - oserver.postWALRestore(ctx, info, (HLogKey)logKey, logEdit); - } else { - legacyWarning(oserver.getClass(), "There are wal keys present that are not HLogKey."); - } - } else { - oserver.postWALRestore(ctx, info, logKey, logEdit); - } + oserver.postWALRestore(ctx, info, logKey, logEdit); } }); } /** - * @deprecated use {@link #postWALRestore(HRegionInfo, WALKey, WALEdit)}; as of 2.0, remove in 3.0 - */ - @Deprecated - public void postWALRestore(final HRegionInfo info, final HLogKey logKey, final WALEdit logEdit) - throws IOException { - postWALRestore(info, (WALKey)logKey, logEdit); - } - - /** * @param familyPaths pairs of { CF, file path } submitted for bulk load * @return true if the default operation should be bypassed * @throws IOException diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index b4f0a29..de5c588 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -661,12 +661,8 @@ public class FSHLog extends AbstractFSWAL { } finally { rollWriterLock.unlock(); } - try { - if (lowReplication || writer != null && writer.getLength() > logrollsize) { - requestLogRoll(lowReplication); - } - } catch (IOException e) { - LOG.warn("Writer.getLength() failed; continuing", e); + if (lowReplication || writer != null && writer.getLength() > logrollsize) { + requestLogRoll(lowReplication); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java deleted file mode 100644 index d7bf4a5..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java +++ /dev/null @@ -1,285 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver.wal; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.EOFException; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.NavigableMap; -import java.util.UUID; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; - -import com.google.common.annotations.VisibleForTesting; - -/** - * A Key for an entry in the change log. - * - * The log intermingles edits to many tables and rows, so each log entry - * identifies the appropriate table and row. Within a table and row, they're - * also sorted. - * - *

Some Transactional edits (START, COMMIT, ABORT) will not have an - * associated row. - * @deprecated use WALKey. Deprecated as of 1.0 (HBASE-12522). Remove in 2.0 - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION) -@Deprecated -public class HLogKey extends WALKey implements Writable { - private static final Log LOG = LogFactory.getLog(HLogKey.class); - - public HLogKey() { - super(); - } - - @VisibleForTesting - public HLogKey(final byte[] encodedRegionName, final TableName tablename, long logSeqNum, - final long now, UUID clusterId) { - super(encodedRegionName, tablename, logSeqNum, now, clusterId); - } - - public HLogKey(final byte[] encodedRegionName, final TableName tablename) { - super(encodedRegionName, tablename, null); - } - - @VisibleForTesting - public HLogKey(final byte[] encodedRegionName, final TableName tablename, final long now) { - super(encodedRegionName, tablename, now); - } - - @VisibleForTesting - public HLogKey(final byte[] encodedRegionName, final TableName tablename, final long now, - final NavigableMap replicationScope) { - super(encodedRegionName, tablename, now, replicationScope); - } - - public HLogKey(final byte[] encodedRegionName, final TableName tablename, final long now, - final MultiVersionConcurrencyControl mvcc, final NavigableMap scopes) { - super(encodedRegionName, tablename, now, mvcc, scopes); - } - - /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - *

Used by log splitting and snapshots. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - name of table - * @param logSeqNum - log sequence number - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) - */ - public HLogKey( - final byte[] encodedRegionName, - final TableName tablename, - long logSeqNum, - final long now, - List clusterIds, - long nonceGroup, - long nonce, - MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc); - } - - /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - *

Used by log splitting and snapshots. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - name of table - * @param logSeqNum - log sequence number - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) - * @param nonceGroup the noncegroup - * @param nonce the nonce - * @param replicationScope the replicationScope of the non-default column families' of the region - */ - public HLogKey( - final byte[] encodedRegionName, - final TableName tablename, - long logSeqNum, - final long now, - List clusterIds, - long nonceGroup, - long nonce, - MultiVersionConcurrencyControl mvcc, NavigableMap replicationScope) { - super(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc, - replicationScope); - } - - /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) - * @param nonceGroup - * @param nonce - */ - public HLogKey(final byte[] encodedRegionName, - final TableName tablename, - final long now, - List clusterIds, - long nonceGroup, - long nonce, - final MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, now, clusterIds, nonceGroup, nonce, mvcc); - } - - /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - * @param logSeqNum - * @param nonceGroup - * @param nonce - */ - public HLogKey(final byte [] encodedRegionName, final TableName tablename, long logSeqNum, - long nonceGroup, long nonce, MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, logSeqNum, nonceGroup, nonce, mvcc); - } - - /** - * @deprecated Don't use these Writables methods. Use PB instead. - */ - @Override - @Deprecated - public void write(DataOutput out) throws IOException { - LOG.warn("HLogKey is being serialized to writable - only expected in test code"); - WritableUtils.writeVInt(out, VERSION.code); - if (compressionContext == null) { - Bytes.writeByteArray(out, this.encodedRegionName); - Bytes.writeByteArray(out, this.tablename.getName()); - } else { - Compressor.writeCompressed(this.encodedRegionName, 0, - this.encodedRegionName.length, out, - compressionContext.regionDict); - Compressor.writeCompressed(this.tablename.getName(), 0, - this.tablename.getName().length, out, - compressionContext.tableDict); - } - out.writeLong(getSequenceId()); - out.writeLong(this.writeTime); - // Don't need to write the clusters information as we are using protobufs from 0.95 - // Writing only the first clusterId for testing the legacy read - Iterator iterator = clusterIds.iterator(); - if(iterator.hasNext()){ - out.writeBoolean(true); - UUID clusterId = iterator.next(); - out.writeLong(clusterId.getMostSignificantBits()); - out.writeLong(clusterId.getLeastSignificantBits()); - } else { - out.writeBoolean(false); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - Version version = Version.UNVERSIONED; - // HLogKey was not versioned in the beginning. - // In order to introduce it now, we make use of the fact - // that encodedRegionName was written with Bytes.writeByteArray, - // which encodes the array length as a vint which is >= 0. - // Hence if the vint is >= 0 we have an old version and the vint - // encodes the length of encodedRegionName. - // If < 0 we just read the version and the next vint is the length. - // @see Bytes#readByteArray(DataInput) - serializeReplicationScope(false); // writable HLogKey does not contain scopes - int len = WritableUtils.readVInt(in); - byte[] tablenameBytes = null; - if (len < 0) { - // what we just read was the version - version = Version.fromCode(len); - // We only compress V2 of WALkey. - // If compression is on, the length is handled by the dictionary - if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) { - len = WritableUtils.readVInt(in); - } - } - if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) { - this.encodedRegionName = new byte[len]; - in.readFully(this.encodedRegionName); - tablenameBytes = Bytes.readByteArray(in); - } else { - this.encodedRegionName = Compressor.readCompressed(in, compressionContext.regionDict); - tablenameBytes = Compressor.readCompressed(in, compressionContext.tableDict); - } - - setSequenceId(in.readLong()); - this.writeTime = in.readLong(); - - this.clusterIds.clear(); - if (version.atLeast(Version.INITIAL)) { - if (in.readBoolean()) { - // read the older log - // Definitely is the originating cluster - clusterIds.add(new UUID(in.readLong(), in.readLong())); - } - } else { - try { - // dummy read (former byte cluster id) - in.readByte(); - } catch(EOFException e) { - // Means it's a very old key, just continue - if (LOG.isTraceEnabled()) LOG.trace(e); - } - } - try { - this.tablename = TableName.valueOf(tablenameBytes); - } catch (IllegalArgumentException iae) { - if (Bytes.toString(tablenameBytes).equals(TableName.OLD_META_STR)) { - // It is a pre-namespace meta table edit, continue with new format. - LOG.info("Got an old .META. edit, continuing with new format "); - this.tablename = TableName.META_TABLE_NAME; - this.encodedRegionName = HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); - } else if (Bytes.toString(tablenameBytes).equals(TableName.OLD_ROOT_STR)) { - this.tablename = TableName.OLD_ROOT_TABLE_NAME; - throw iae; - } else throw iae; - } - // Do not need to read the clusters information as we are using protobufs from 0.95 - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java index 2114cc4..b2d6751 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java @@ -23,18 +23,18 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.util.LRUDictionary; import org.apache.hadoop.hbase.util.FSUtils; - import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL.Entry; +import org.apache.hadoop.hbase.wal.WALKey; @InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.PHOENIX}) public abstract class ReaderBase implements AbstractFSWALProvider.Reader { @@ -91,9 +91,7 @@ public abstract class ReaderBase implements AbstractFSWALProvider.Reader { public Entry next(Entry reuse) throws IOException { Entry e = reuse; if (e == null) { - // we use HLogKey here instead of WALKey directly to support legacy coprocessors, - // seqencefile based readers, and HLogInputFormat. - e = new Entry(new HLogKey(), new WALEdit()); + e = new Entry(new WALKey(), new WALEdit()); } if (compressionContext != null) { e.setCompressionContext(compressionContext); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayHLogKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayHLogKey.java deleted file mode 100644 index d5a1561..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayHLogKey.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver.wal; - -import java.util.List; -import java.util.UUID; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; - -/** - * An HLogKey specific to WalEdits coming from replay. - */ -@InterfaceAudience.Private -public class ReplayHLogKey extends HLogKey { - - public ReplayHLogKey(final byte [] encodedRegionName, final TableName tablename, - final long now, List clusterIds, long nonceGroup, long nonce, - MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, now, clusterIds, nonceGroup, nonce, mvcc); - } - - public ReplayHLogKey(final byte [] encodedRegionName, final TableName tablename, - long logSeqNum, final long now, List clusterIds, long nonceGroup, long nonce, - MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc); - } - - /** - * Returns the original sequence id - * @return long the new assigned sequence number - */ - @Override - public long getSequenceId() { - return this.getOrigLogSeqNum(); - } -} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java deleted file mode 100644 index e41e1c3..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java +++ /dev/null @@ -1,309 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver.wal; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.NavigableMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.wal.WAL.Entry; -import org.apache.hadoop.hdfs.client.HdfsDataInputStream; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.Metadata; -import org.apache.hadoop.io.Text; - -@InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.PHOENIX, - HBaseInterfaceAudience.CONFIG}) -public class SequenceFileLogReader extends ReaderBase { - private static final Log LOG = LogFactory.getLog(SequenceFileLogReader.class); - - // Legacy stuff from pre-PB WAL metadata. - private static final Text WAL_VERSION_KEY = new Text("version"); - // Let the version be 1. Let absence of a version meta tag be old, version 0. - // Set this version '1' to be the version that introduces compression, - // the COMPRESSION_VERSION. - private static final int COMPRESSION_VERSION = 1; - private static final Text WAL_COMPRESSION_TYPE_KEY = new Text("compression.type"); - private static final Text DICTIONARY_COMPRESSION_TYPE = new Text("dictionary"); - - /** - * Hack just to set the correct file length up in SequenceFile.Reader. - * See HADOOP-6307. The below is all about setting the right length on the - * file we are reading. fs.getFileStatus(file).getLen() is passed down to - * a private SequenceFile.Reader constructor. This won't work. Need to do - * the available on the stream. The below is ugly. It makes getPos, the - * first time its called, return length of the file -- i.e. tell a lie -- just - * so this line up in SF.Reader's constructor ends up with right answer: - * - * this.end = in.getPos() + length; - * - */ - private static class WALReader extends SequenceFile.Reader { - - WALReader(final FileSystem fs, final Path p, final Configuration c) - throws IOException { - super(fs, p, c); - } - - @Override - protected FSDataInputStream openFile(FileSystem fs, Path file, - int bufferSize, long length) - throws IOException { - return new WALReaderFSDataInputStream(super.openFile(fs, file, - bufferSize, length), length); - } - - /** - * Override just so can intercept first call to getPos. - */ - static class WALReaderFSDataInputStream extends FSDataInputStream { - private boolean firstGetPosInvocation = true; - private long length; - - WALReaderFSDataInputStream(final FSDataInputStream is, final long l) - throws IOException { - super(is); - this.length = l; - } - - // This section can be confusing. It is specific to how HDFS works. - // Let me try to break it down. This is the problem: - // - // 1. HDFS DataNodes update the NameNode about a filename's length - // on block boundaries or when a file is closed. Therefore, - // if an RS dies, then the NN's fs.getLength() can be out of date - // 2. this.in.available() would work, but it returns int & - // therefore breaks for files > 2GB (happens on big clusters) - // 3. DFSInputStream.getFileLength() gets the actual length from the DNs - // 4. DFSInputStream is wrapped 2 levels deep : this.in.in - // - // So, here we adjust getPos() using getFileLength() so the - // SequenceFile.Reader constructor (aka: first invocation) comes out - // with the correct end of the file: - // this.end = in.getPos() + length; - @Override - public long getPos() throws IOException { - if (this.firstGetPosInvocation) { - this.firstGetPosInvocation = false; - long adjust = 0; - HdfsDataInputStream hdfsDataInputStream = null; - try { - if (this.in.getClass().getName().endsWith("HdfsDataInputStream") - || this.in.getClass().getName().endsWith("DFSInputStream")) { - hdfsDataInputStream = (HdfsDataInputStream) this.getWrappedStream(); - long realLength = hdfsDataInputStream.getVisibleLength(); - assert(realLength >= this.length); - adjust = realLength - this.length; - } else { - LOG.info( - "Input stream class: " + this.in.getClass().getName() + ", not adjusting length"); - } - } catch (Exception e) { - LOG.warn("Error while trying to get accurate file length. " - + "Truncation / data loss may occur if RegionServers die.", - e); - throw new IOException(e); - } - return adjust + super.getPos(); - } - return super.getPos(); - } - } - } - - // Protected for tests. - protected SequenceFile.Reader reader; - long entryStart = 0; // needed for logging exceptions - - public SequenceFileLogReader() { - super(); - } - - @Override - public void close() throws IOException { - try { - if (reader != null) { - this.reader.close(); - this.reader = null; - } - } catch (IOException ioe) { - throw addFileInfoToException(ioe); - } - } - - @Override - public long getPosition() throws IOException { - return reader != null ? reader.getPosition() : 0; - } - - @Override - public void reset() throws IOException { - // Resetting the reader lets us see newly added data if the file is being written to - // We also keep the same compressionContext which was previously populated for this file - reader = new WALReader(fs, path, conf); - } - - @Override - protected String initReader(FSDataInputStream stream) throws IOException { - // We don't use the stream because we have to have the magic stream above. - if (stream != null) { - stream.close(); - } - reset(); - return null; - } - - @Override - protected void initAfterCompression(String cellCodecClsName) throws IOException { - // Nothing to do here - } - - @Override - protected void initAfterCompression() throws IOException { - // Nothing to do here - } - - @Override - protected boolean hasCompression() { - return isWALCompressionEnabled(reader.getMetadata()); - } - - @Override - protected boolean hasTagCompression() { - // Tag compression not supported with old SequenceFileLog Reader/Writer - return false; - } - - /** - * Call this method after init() has been executed - * @return whether WAL compression is enabled - */ - static boolean isWALCompressionEnabled(final Metadata metadata) { - // Check version is >= VERSION? - Text txt = metadata.get(WAL_VERSION_KEY); - if (txt == null || Integer.parseInt(txt.toString()) < COMPRESSION_VERSION) { - return false; - } - // Now check that compression type is present. Currently only one value. - txt = metadata.get(WAL_COMPRESSION_TYPE_KEY); - return txt != null && txt.equals(DICTIONARY_COMPRESSION_TYPE); - } - - - /** - * fill in the passed entry with teh next key/value. - * Note that because this format deals with our legacy storage, the provided - * Entery MUST use an {@link HLogKey} for the key. - * @return boolean indicating if the contents of Entry have been filled in. - */ - @Override - protected boolean readNext(Entry e) throws IOException { - try { - if (!(e.getKey() instanceof HLogKey)) { - final IllegalArgumentException exception = new IllegalArgumentException( - "SequenceFileLogReader only works when given entries that have HLogKey for keys. This" + - " one had '" + e.getKey().getClass() + "'"); - LOG.error("We need to use the legacy SequenceFileLogReader to handle a " + - " pre-0.96 style WAL, but HBase internals failed to use the deprecated HLogKey class." + - " This is a bug; please file an issue or email the developer mailing list. You will " + - "need the following exception details when seeking help from the HBase community.", - exception); - throw exception; - } - boolean hasNext = this.reader.next((HLogKey)e.getKey(), e.getEdit()); - if (!hasNext) return false; - // Scopes are probably in WAL edit, move to key - NavigableMap scopes = e.getEdit().getAndRemoveScopes(); - if (scopes != null) { - e.getKey().readOlderScopes(scopes); - } - return true; - } catch (IOException ioe) { - throw addFileInfoToException(ioe); - } - } - - @Override - protected void seekOnFs(long pos) throws IOException { - try { - reader.seek(pos); - } catch (IOException ioe) { - throw addFileInfoToException(ioe); - } - } - - protected IOException addFileInfoToException(final IOException ioe) - throws IOException { - long pos = -1; - try { - pos = getPosition(); - } catch (IOException e) { - LOG.warn("Failed getting position to add to throw", e); - } - - // See what SequenceFile.Reader thinks is the end of the file - long end = Long.MAX_VALUE; - try { - Field fEnd = SequenceFile.Reader.class.getDeclaredField("end"); - fEnd.setAccessible(true); - end = fEnd.getLong(this.reader); - } catch(NoSuchFieldException nfe) { - /* reflection failure, keep going */ - if (LOG.isTraceEnabled()) LOG.trace(nfe); - } catch(IllegalAccessException iae) { - /* reflection failure, keep going */ - if (LOG.isTraceEnabled()) LOG.trace(iae); - } catch(Exception e) { - /* All other cases. Should we handle it more aggressively? */ - LOG.warn("Unexpected exception when accessing the end field", e); - } - - String msg = (this.path == null? "": this.path.toString()) + - ", entryStart=" + entryStart + ", pos=" + pos + - ((end == Long.MAX_VALUE) ? "" : ", end=" + end) + - ", edit=" + this.edit; - - // Enhance via reflection so we don't change the original class type - try { - return (IOException) ioe.getClass() - .getConstructor(String.class) - .newInstance(msg) - .initCause(ioe); - } catch(NoSuchMethodException nfe) { - /* reflection failure, keep going */ - if (LOG.isTraceEnabled()) LOG.trace(nfe); - } catch(IllegalAccessException iae) { - /* reflection failure, keep going */ - if (LOG.isTraceEnabled()) LOG.trace(iae); - } catch(Exception e) { - /* All other cases. Should we handle it more aggressively? */ - LOG.warn("Unexpected exception when accessing the end field", e); - } - return ioe; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java index 87019e8..4d7e868 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java @@ -23,12 +23,15 @@ package org.apache.hadoop.hbase.regionserver.wal; import java.io.IOException; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.coprocessor.*; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.WALCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.WALObserver; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALKey; @@ -48,9 +51,6 @@ public class WALCoprocessorHost private final WAL wal; - final boolean useLegacyPre; - final boolean useLegacyPost; - @Override public WAL getWAL() { return wal; @@ -70,14 +70,6 @@ public class WALCoprocessorHost final WAL wal) { super(impl, priority, seq, conf); this.wal = wal; - // Pick which version of the API we'll call. - // This way we avoid calling the new version on older WALObservers so - // we can maintain binary compatibility. - // See notes in javadoc for WALObserver - useLegacyPre = useLegacyMethod(impl.getClass(), "preWALWrite", ObserverContext.class, - HRegionInfo.class, WALKey.class, WALEdit.class); - useLegacyPost = useLegacyMethod(impl.getClass(), "postWALWrite", ObserverContext.class, - HRegionInfo.class, WALKey.class, WALEdit.class); } } @@ -131,16 +123,7 @@ public class WALCoprocessorHost ClassLoader cl = currentThread.getContextClassLoader(); try { currentThread.setContextClassLoader(env.getClassLoader()); - if (env.useLegacyPre) { - if (logKey instanceof HLogKey) { - observer.preWALWrite(ctx, info, (HLogKey)logKey, logEdit); - } else { - legacyWarning(observer.getClass(), - "There are wal keys present that are not HLogKey."); - } - } else { - observer.preWALWrite(ctx, info, logKey, logEdit); - } + observer.preWALWrite(ctx, info, logKey, logEdit); } catch (Throwable e) { handleCoprocessorThrowable(env, e); } finally { @@ -175,16 +158,7 @@ public class WALCoprocessorHost ClassLoader cl = currentThread.getContextClassLoader(); try { currentThread.setContextClassLoader(env.getClassLoader()); - if (env.useLegacyPost) { - if (logKey instanceof HLogKey) { - observer.postWALWrite(ctx, info, (HLogKey)logKey, logEdit); - } else { - legacyWarning(observer.getClass(), - "There are wal keys present that are not HLogKey."); - } - } else { - observer.postWALWrite(ctx, info, logKey, logEdit); - } + observer.postWALWrite(ctx, info, logKey, logEdit); } catch (Throwable e) { handleCoprocessorThrowable(env, e); } finally { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java index 1794464..cbdcea9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java @@ -18,6 +18,10 @@ package org.apache.hadoop.hbase.security.access; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; + import java.io.ByteArrayInputStream; import java.io.DataInput; import java.io.DataInputStream; @@ -39,7 +43,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; @@ -70,10 +73,9 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.Text; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Lists; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableFactories; +import org.apache.hadoop.io.WritableUtils; /** * Maintains lists of permission grants to users and groups to allow for @@ -596,13 +598,36 @@ public class AccessControlLists { return ProtobufUtil.prependPBMagic(AccessControlUtil.toUserTablePermissions(perms).toByteArray()); } + // This is part of the old HbaseObjectWritableFor96Migration. + private static final int LIST_CODE = 61; + + private static final int WRITABLE_CODE = 14; + + private static final int WRITABLE_NOT_ENCODED = 0; + + private static List readWritablePermissions(DataInput in, Configuration conf) + throws IOException, ClassNotFoundException { + assert WritableUtils.readVInt(in) == LIST_CODE; + int length = in.readInt(); + List list = new ArrayList<>(length); + for (int i = 0; i < length; i++) { + assert WritableUtils.readVInt(in) == WRITABLE_CODE; + assert WritableUtils.readVInt(in) == WRITABLE_NOT_ENCODED; + String className = Text.readString(in); + Class clazz = conf.getClassByName(className).asSubclass(Writable.class); + Writable instance = WritableFactories.newInstance(clazz, conf); + instance.readFields(in); + list.add((TablePermission) instance); + } + return list; + } + /** - * Reads a set of permissions as {@link org.apache.hadoop.io.Writable} instances - * from the input stream. + * Reads a set of permissions as {@link org.apache.hadoop.io.Writable} instances from the input + * stream. */ public static ListMultimap readPermissions(byte[] data, - Configuration conf) - throws DeserializationException { + Configuration conf) throws DeserializationException { if (ProtobufUtil.isPBMagicPrefix(data)) { int pblen = ProtobufUtil.lengthOfPBMagic(); try { @@ -614,17 +639,18 @@ public class AccessControlLists { throw new DeserializationException(e); } } else { - ListMultimap perms = ArrayListMultimap.create(); + // TODO: We have to re-write non-PB data as PB encoded. Otherwise we will carry old Writables + // forever (here and a couple of other places). + ListMultimap perms = ArrayListMultimap.create(); try { DataInput in = new DataInputStream(new ByteArrayInputStream(data)); int length = in.readInt(); - for (int i=0; i userPerms = - (List)HbaseObjectWritableFor96Migration.readObject(in, conf); + List userPerms = readWritablePermissions(in, conf); perms.putAll(user, userPerms); } - } catch (IOException e) { + } catch (IOException | ClassNotFoundException e) { throw new DeserializationException(e); } return perms; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java deleted file mode 100644 index d8c8850..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java +++ /dev/null @@ -1,844 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.security.access; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.lang.reflect.Array; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NavigableSet; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.hbase.ClusterStatus; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Action; -import org.apache.hadoop.hbase.client.Append; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Increment; -import org.apache.hadoop.hbase.client.MultiAction; -import org.apache.hadoop.hbase.client.MultiResponse; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Row; -import org.apache.hadoop.hbase.client.RowMutations; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.filter.BinaryComparator; -import org.apache.hadoop.hbase.filter.BitComparator; -import org.apache.hadoop.hbase.filter.ByteArrayComparable; -import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; -import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; -import org.apache.hadoop.hbase.filter.ColumnRangeFilter; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; -import org.apache.hadoop.hbase.filter.DependentColumnFilter; -import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; -import org.apache.hadoop.hbase.filter.InclusiveStopFilter; -import org.apache.hadoop.hbase.filter.KeyOnlyFilter; -import org.apache.hadoop.hbase.filter.PageFilter; -import org.apache.hadoop.hbase.filter.PrefixFilter; -import org.apache.hadoop.hbase.filter.QualifierFilter; -import org.apache.hadoop.hbase.filter.RandomRowFilter; -import org.apache.hadoop.hbase.filter.RowFilter; -import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; -import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; -import org.apache.hadoop.hbase.filter.SkipFilter; -import org.apache.hadoop.hbase.filter.ValueFilter; -import org.apache.hadoop.hbase.filter.WhileMatchFilter; -import org.apache.hadoop.hbase.io.WritableWithSize; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; -import org.apache.hadoop.hbase.regionserver.RegionOpeningState; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.regionserver.wal.WALEdit; -import org.apache.hadoop.hbase.wal.WAL.Entry; -import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.ProtoUtil; -import org.apache.hadoop.io.DataOutputOutputStream; -import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableFactories; -import org.apache.hadoop.io.WritableUtils; - -import com.google.protobuf.Message; -import com.google.protobuf.RpcController; - -/** - *

This is a customized version of the polymorphic hadoop - * {@link ObjectWritable}. It removes UTF8 (HADOOP-414). - * Using {@link Text} intead of UTF-8 saves ~2% CPU between reading and writing - * objects running a short sequentialWrite Performance Evaluation test just in - * ObjectWritable alone; more when we're doing randomRead-ing. Other - * optimizations include our passing codes for classes instead of the - * actual class names themselves. This makes it so this class needs amendment - * if non-Writable classes are introduced -- if passed a Writable for which we - * have no code, we just do the old-school passing of the class name, etc. -- - * but passing codes the savings are large particularly when cell - * data is small (If < a couple of kilobytes, the encoding/decoding of class - * name and reflection to instantiate class was costing in excess of the cell - * handling). - * @deprecated This class is needed migrating TablePermissions written with - * Writables. It is needed to read old permissions written pre-0.96. This - * class is to be removed after HBase 0.96 ships since then all permissions - * will have been migrated and written with protobufs. - */ -@Deprecated -@InterfaceAudience.Private -class HbaseObjectWritableFor96Migration implements Writable, WritableWithSize, Configurable { - private final static Log LOG = LogFactory.getLog(HbaseObjectWritableFor96Migration.class); - - // Here we maintain two static maps of classes to code and vice versa. - // Add new classes+codes as wanted or figure way to auto-generate these - // maps. - static final Map> CODE_TO_CLASS = - new HashMap>(); - static final Map, Integer> CLASS_TO_CODE = - new HashMap, Integer>(); - // Special code that means 'not-encoded'; in this case we do old school - // sending of the class name using reflection, etc. - private static final byte NOT_ENCODED = 0; - //Generic array means that the array type is not one of the pre-defined arrays - //in the CLASS_TO_CODE map, but we have to still encode the array since it's - //elements are serializable by this class. - private static final int GENERIC_ARRAY_CODE; - private static final int NEXT_CLASS_CODE; - static { - //////////////////////////////////////////////////////////////////////////// - // WARNING: Please do not insert, remove or swap any line in this static // - // block. Doing so would change or shift all the codes used to serialize // - // objects, which makes backwards compatibility very hard for clients. // - // New codes should always be added at the end. Code removal is // - // discouraged because code is a short now. // - //////////////////////////////////////////////////////////////////////////// - - int code = NOT_ENCODED + 1; - // Primitive types. - addToMap(Boolean.TYPE, code++); - addToMap(Byte.TYPE, code++); - addToMap(Character.TYPE, code++); - addToMap(Short.TYPE, code++); - addToMap(Integer.TYPE, code++); - addToMap(Long.TYPE, code++); - addToMap(Float.TYPE, code++); - addToMap(Double.TYPE, code++); - addToMap(Void.TYPE, code++); - - // Other java types - addToMap(String.class, code++); - addToMap(byte [].class, code++); - addToMap(byte [][].class, code++); - - // Hadoop types - addToMap(Text.class, code++); - addToMap(Writable.class, code++); - addToMap(Writable [].class, code++); - code++; // Removed - addToMap(NullInstance.class, code++); - - // Hbase types - addToMap(HColumnDescriptor.class, code++); - addToMap(HConstants.Modify.class, code++); - - // We used to have a class named HMsg but its been removed. Rather than - // just axe it, use following random Integer class -- we just chose any - // class from java.lang -- instead just so codes that follow stay - // in same relative place. - addToMap(Integer.class, code++); - addToMap(Integer[].class, code++); - - //HRegion shouldn't be pushed across the wire. - code++; //addToMap(HRegion.class, code++); - code++; //addToMap(HRegion[].class, code++); - - addToMap(HRegionInfo.class, code++); - addToMap(HRegionInfo[].class, code++); - code++; // Removed - code++; // Removed - addToMap(HTableDescriptor.class, code++); - addToMap(MapWritable.class, code++); - - // - // HBASE-880 - // - addToMap(ClusterStatus.class, code++); - addToMap(Delete.class, code++); - addToMap(Get.class, code++); - addToMap(KeyValue.class, code++); - addToMap(KeyValue[].class, code++); - addToMap(Put.class, code++); - addToMap(Put[].class, code++); - addToMap(Result.class, code++); - addToMap(Result[].class, code++); - addToMap(Scan.class, code++); - - addToMap(WhileMatchFilter.class, code++); - addToMap(PrefixFilter.class, code++); - addToMap(PageFilter.class, code++); - addToMap(InclusiveStopFilter.class, code++); - addToMap(ColumnCountGetFilter.class, code++); - addToMap(SingleColumnValueFilter.class, code++); - addToMap(SingleColumnValueExcludeFilter.class, code++); - addToMap(BinaryComparator.class, code++); - addToMap(BitComparator.class, code++); - addToMap(CompareFilter.class, code++); - addToMap(RowFilter.class, code++); - addToMap(ValueFilter.class, code++); - addToMap(QualifierFilter.class, code++); - addToMap(SkipFilter.class, code++); - addToMap(ByteArrayComparable.class, code++); - addToMap(FirstKeyOnlyFilter.class, code++); - addToMap(DependentColumnFilter.class, code++); - - addToMap(Delete [].class, code++); - - addToMap(Entry.class, code++); - addToMap(Entry[].class, code++); - addToMap(HLogKey.class, code++); - - addToMap(List.class, code++); - - addToMap(NavigableSet.class, code++); - addToMap(ColumnPrefixFilter.class, code++); - - // Multi - addToMap(Row.class, code++); - addToMap(Action.class, code++); - addToMap(MultiAction.class, code++); - addToMap(MultiResponse.class, code++); - - // coprocessor execution - // Exec no longer exists --> addToMap(Exec.class, code++); - code++; - addToMap(Increment.class, code++); - - addToMap(KeyOnlyFilter.class, code++); - - // serializable - addToMap(Serializable.class, code++); - - addToMap(RandomRowFilter.class, code++); - - addToMap(CompareOp.class, code++); - - addToMap(ColumnRangeFilter.class, code++); - - // HServerLoad no longer exists; increase code so other classes stay the same. - code++; - //addToMap(HServerLoad.class, code++); - - addToMap(RegionOpeningState.class, code++); - - addToMap(HTableDescriptor[].class, code++); - - addToMap(Append.class, code++); - - addToMap(RowMutations.class, code++); - - addToMap(Message.class, code++); - - //java.lang.reflect.Array is a placeholder for arrays not defined above - GENERIC_ARRAY_CODE = code++; - addToMap(Array.class, GENERIC_ARRAY_CODE); - - addToMap(RpcController.class, code++); - - // make sure that this is the last statement in this static block - NEXT_CLASS_CODE = code; - } - - private Class declaredClass; - private Object instance; - private Configuration conf; - - /** default constructor for writable */ - HbaseObjectWritableFor96Migration() { - super(); - } - - /** - * @param instance - */ - HbaseObjectWritableFor96Migration(Object instance) { - set(instance); - } - - /** - * @param declaredClass - * @param instance - */ - HbaseObjectWritableFor96Migration(Class declaredClass, Object instance) { - this.declaredClass = declaredClass; - this.instance = instance; - } - - /** @return the instance, or null if none. */ - Object get() { return instance; } - - /** @return the class this is meant to be. */ - Class getDeclaredClass() { return declaredClass; } - - /** - * Reset the instance. - * @param instance - */ - void set(Object instance) { - this.declaredClass = instance.getClass(); - this.instance = instance; - } - - /** - * @see java.lang.Object#toString() - */ - @Override - public String toString() { - return "OW[class=" + declaredClass + ",value=" + instance + "]"; - } - - - public void readFields(DataInput in) throws IOException { - readObject(in, this, this.conf); - } - - public void write(DataOutput out) throws IOException { - writeObject(out, instance, declaredClass, conf); - } - - public long getWritableSize() { - return getWritableSize(instance, declaredClass, conf); - } - - private static class NullInstance extends Configured implements Writable { - Class declaredClass; - /** default constructor for writable */ - @SuppressWarnings("unused") - public NullInstance() { super(null); } - - /** - * @param declaredClass - * @param conf - */ - public NullInstance(Class declaredClass, Configuration conf) { - super(conf); - this.declaredClass = declaredClass; - } - - public void readFields(DataInput in) throws IOException { - this.declaredClass = CODE_TO_CLASS.get(WritableUtils.readVInt(in)); - } - - public void write(DataOutput out) throws IOException { - writeClassCode(out, this.declaredClass); - } - } - - static Integer getClassCode(final Class c) - throws IOException { - Integer code = CLASS_TO_CODE.get(c); - if (code == null ) { - if (List.class.isAssignableFrom(c)) { - code = CLASS_TO_CODE.get(List.class); - } else if (Writable.class.isAssignableFrom(c)) { - code = CLASS_TO_CODE.get(Writable.class); - } else if (c.isArray()) { - code = CLASS_TO_CODE.get(Array.class); - } else if (Message.class.isAssignableFrom(c)) { - code = CLASS_TO_CODE.get(Message.class); - } else if (Serializable.class.isAssignableFrom(c)){ - code = CLASS_TO_CODE.get(Serializable.class); - } else if (Scan.class.isAssignableFrom(c)) { - code = CLASS_TO_CODE.get(Scan.class); - } - } - return code; - } - - /** - * @return the next object code in the list. Used in testing to verify that additional fields are not added - */ - static int getNextClassCode(){ - return NEXT_CLASS_CODE; - } - - /** - * Write out the code for passed Class. - * @param out - * @param c - * @throws IOException - */ - static void writeClassCode(final DataOutput out, final Class c) - throws IOException { - Integer code = getClassCode(c); - - if (code == null) { - LOG.error("Unsupported type " + c); - StackTraceElement[] els = new Exception().getStackTrace(); - for(StackTraceElement elem : els) { - LOG.error(elem.getMethodName()); - } - throw new UnsupportedOperationException("No code for unexpected " + c); - } - WritableUtils.writeVInt(out, code); - } - - static long getWritableSize(Object instance, Class declaredClass, - Configuration conf) { - return 0L; // no hint is the default. - } - /** - * Write a {@link Writable}, {@link String}, primitive type, or an array of - * the preceding. - * @param out - * @param instance - * @param declaredClass - * @param conf - * @throws IOException - */ - @SuppressWarnings("unchecked") - static void writeObject(DataOutput out, Object instance, - Class declaredClass, - Configuration conf) - throws IOException { - - Object instanceObj = instance; - Class declClass = declaredClass; - - if (instanceObj == null) { // null - instanceObj = new NullInstance(declClass, conf); - declClass = Writable.class; - } - writeClassCode(out, declClass); - if (declClass.isArray()) { // array - // If bytearray, just dump it out -- avoid the recursion and - // byte-at-a-time we were previously doing. - if (declClass.equals(byte [].class)) { - Bytes.writeByteArray(out, (byte [])instanceObj); - } else { - //if it is a Generic array, write the element's type - if (getClassCode(declaredClass) == GENERIC_ARRAY_CODE) { - Class componentType = declaredClass.getComponentType(); - writeClass(out, componentType); - } - - int length = Array.getLength(instanceObj); - out.writeInt(length); - for (int i = 0; i < length; i++) { - Object item = Array.get(instanceObj, i); - writeObject(out, item, - item.getClass(), conf); - } - } - } else if (List.class.isAssignableFrom(declClass)) { - List list = (List)instanceObj; - int length = list.size(); - out.writeInt(length); - for (int i = 0; i < length; i++) { - Object elem = list.get(i); - writeObject(out, elem, - elem == null ? Writable.class : elem.getClass(), conf); - } - } else if (declClass == String.class) { // String - Text.writeString(out, (String)instanceObj); - } else if (declClass.isPrimitive()) { // primitive type - if (declClass == Boolean.TYPE) { // boolean - out.writeBoolean(((Boolean)instanceObj).booleanValue()); - } else if (declClass == Character.TYPE) { // char - out.writeChar(((Character)instanceObj).charValue()); - } else if (declClass == Byte.TYPE) { // byte - out.writeByte(((Byte)instanceObj).byteValue()); - } else if (declClass == Short.TYPE) { // short - out.writeShort(((Short)instanceObj).shortValue()); - } else if (declClass == Integer.TYPE) { // int - out.writeInt(((Integer)instanceObj).intValue()); - } else if (declClass == Long.TYPE) { // long - out.writeLong(((Long)instanceObj).longValue()); - } else if (declClass == Float.TYPE) { // float - out.writeFloat(((Float)instanceObj).floatValue()); - } else if (declClass == Double.TYPE) { // double - out.writeDouble(((Double)instanceObj).doubleValue()); - } else if (declClass == Void.TYPE) { // void - } else { - throw new IllegalArgumentException("Not a primitive: "+declClass); - } - } else if (declClass.isEnum()) { // enum - Text.writeString(out, ((Enum)instanceObj).name()); - } else if (Message.class.isAssignableFrom(declaredClass)) { - Text.writeString(out, instanceObj.getClass().getName()); - ((Message)instance).writeDelimitedTo( - DataOutputOutputStream.constructOutputStream(out)); - } else if (Writable.class.isAssignableFrom(declClass)) { // Writable - Class c = instanceObj.getClass(); - Integer code = CLASS_TO_CODE.get(c); - if (code == null) { - out.writeByte(NOT_ENCODED); - Text.writeString(out, c.getName()); - } else { - writeClassCode(out, c); - } - ((Writable)instanceObj).write(out); - } else if (Serializable.class.isAssignableFrom(declClass)) { - Class c = instanceObj.getClass(); - Integer code = CLASS_TO_CODE.get(c); - if (code == null) { - out.writeByte(NOT_ENCODED); - Text.writeString(out, c.getName()); - } else { - writeClassCode(out, c); - } - ByteArrayOutputStream bos = null; - ObjectOutputStream oos = null; - try{ - bos = new ByteArrayOutputStream(); - oos = new ObjectOutputStream(bos); - oos.writeObject(instanceObj); - byte[] value = bos.toByteArray(); - out.writeInt(value.length); - out.write(value); - } finally { - if(bos!=null) bos.close(); - if(oos!=null) oos.close(); - } - } else if (Scan.class.isAssignableFrom(declClass)) { - Scan scan = (Scan)instanceObj; - byte [] scanBytes = ProtobufUtil.toScan(scan).toByteArray(); - out.writeInt(scanBytes.length); - out.write(scanBytes); - } else if (Entry.class.isAssignableFrom(declClass)) { - // Entry is no longer Writable, maintain compatible serialization. - // Writables write their exact runtime class - Class c = instanceObj.getClass(); - Integer code = CLASS_TO_CODE.get(c); - if (code == null) { - out.writeByte(NOT_ENCODED); - Text.writeString(out, c.getName()); - } else { - writeClassCode(out, c); - } - final Entry entry = (Entry)instanceObj; - // We only support legacy HLogKey - WALKey key = entry.getKey(); - if (!(key instanceof HLogKey)) { - throw new IOException("Can't write Entry '" + instanceObj + "' due to key class '" + - key.getClass() + "'"); - } - ((HLogKey)key).write(out); - entry.getEdit().write(out); - } else { - throw new IOException("Can't write: "+instanceObj+" as "+declClass); - } - } - - /** Writes the encoded class code as defined in CLASS_TO_CODE, or - * the whole class name if not defined in the mapping. - */ - static void writeClass(DataOutput out, Class c) throws IOException { - Integer code = CLASS_TO_CODE.get(c); - if (code == null) { - WritableUtils.writeVInt(out, NOT_ENCODED); - Text.writeString(out, c.getName()); - } else { - WritableUtils.writeVInt(out, code); - } - } - - /** Reads and returns the class as written by {@link #writeClass(DataOutput, Class)} */ - static Class readClass(Configuration conf, DataInput in) throws IOException { - Class instanceClass = null; - int b = (byte)WritableUtils.readVInt(in); - if (b == NOT_ENCODED) { - String className = Text.readString(in); - try { - instanceClass = getClassByName(conf, className); - } catch (ClassNotFoundException e) { - LOG.error("Can't find class " + className, e); - throw new IOException("Can't find class " + className, e); - } - } else { - instanceClass = CODE_TO_CLASS.get(b); - } - return instanceClass; - } - - /** - * Read a {@link Writable}, {@link String}, primitive type, or an array of - * the preceding. - * @param in - * @param conf - * @return the object - * @throws IOException - */ - static Object readObject(DataInput in, Configuration conf) - throws IOException { - return readObject(in, null, conf); - } - - /** - * Read a {@link Writable}, {@link String}, primitive type, or an array of - * the preceding. - * @param in - * @param objectWritable - * @param conf - * @return the object - * @throws IOException - */ - @SuppressWarnings("unchecked") - static Object readObject(DataInput in, - HbaseObjectWritableFor96Migration objectWritable, Configuration conf) - throws IOException { - Class declaredClass = CODE_TO_CLASS.get(WritableUtils.readVInt(in)); - Object instance; - if (declaredClass.isPrimitive()) { // primitive types - if (declaredClass == Boolean.TYPE) { // boolean - instance = Boolean.valueOf(in.readBoolean()); - } else if (declaredClass == Character.TYPE) { // char - instance = Character.valueOf(in.readChar()); - } else if (declaredClass == Byte.TYPE) { // byte - instance = Byte.valueOf(in.readByte()); - } else if (declaredClass == Short.TYPE) { // short - instance = Short.valueOf(in.readShort()); - } else if (declaredClass == Integer.TYPE) { // int - instance = Integer.valueOf(in.readInt()); - } else if (declaredClass == Long.TYPE) { // long - instance = Long.valueOf(in.readLong()); - } else if (declaredClass == Float.TYPE) { // float - instance = Float.valueOf(in.readFloat()); - } else if (declaredClass == Double.TYPE) { // double - instance = Double.valueOf(in.readDouble()); - } else if (declaredClass == Void.TYPE) { // void - instance = null; - } else { - throw new IllegalArgumentException("Not a primitive: "+declaredClass); - } - } else if (declaredClass.isArray()) { // array - if (declaredClass.equals(byte [].class)) { - instance = Bytes.readByteArray(in); - } else { - int length = in.readInt(); - instance = Array.newInstance(declaredClass.getComponentType(), length); - for (int i = 0; i < length; i++) { - Array.set(instance, i, readObject(in, conf)); - } - } - } else if (declaredClass.equals(Array.class)) { //an array not declared in CLASS_TO_CODE - Class componentType = readClass(conf, in); - int length = in.readInt(); - instance = Array.newInstance(componentType, length); - for (int i = 0; i < length; i++) { - Array.set(instance, i, readObject(in, conf)); - } - } else if (List.class.isAssignableFrom(declaredClass)) { // List - int length = in.readInt(); - instance = new ArrayList(length); - for (int i = 0; i < length; i++) { - ((ArrayList)instance).add(readObject(in, conf)); - } - } else if (declaredClass == String.class) { // String - instance = Text.readString(in); - } else if (declaredClass.isEnum()) { // enum - instance = Enum.valueOf((Class) declaredClass, - Text.readString(in)); - } else if (declaredClass == Message.class) { - String className = Text.readString(in); - try { - declaredClass = getClassByName(conf, className); - instance = tryInstantiateProtobuf(declaredClass, in); - } catch (ClassNotFoundException e) { - LOG.error("Can't find class " + className, e); - throw new IOException("Can't find class " + className, e); - } - } else if (Scan.class.isAssignableFrom(declaredClass)) { - int length = in.readInt(); - byte [] scanBytes = new byte[length]; - in.readFully(scanBytes); - ClientProtos.Scan.Builder scanProto = ClientProtos.Scan.newBuilder(); - ProtobufUtil.mergeFrom(scanProto, scanBytes); - instance = ProtobufUtil.toScan(scanProto.build()); - } else { // Writable or Serializable - Class instanceClass = null; - int b = (byte)WritableUtils.readVInt(in); - if (b == NOT_ENCODED) { - String className = Text.readString(in); - if ("org.apache.hadoop.hbase.regionserver.wal.HLog$Entry".equals(className)) { - className = Entry.class.getName(); - } - try { - instanceClass = getClassByName(conf, className); - } catch (ClassNotFoundException e) { - LOG.error("Can't find class " + className, e); - throw new IOException("Can't find class " + className, e); - } - } else { - instanceClass = CODE_TO_CLASS.get(b); - } - if(Writable.class.isAssignableFrom(instanceClass)){ - Writable writable = WritableFactories.newInstance(instanceClass, conf); - try { - writable.readFields(in); - } catch (Exception e) { - LOG.error("Error in readFields", e); - throw new IOException("Error in readFields" , e); - } - instance = writable; - if (instanceClass == NullInstance.class) { // null - declaredClass = ((NullInstance)instance).declaredClass; - instance = null; - } - } else if (Entry.class.isAssignableFrom(instanceClass)) { - // Entry stopped being Writable; maintain serialization support. - final HLogKey key = new HLogKey(); - final WALEdit edit = new WALEdit(); - key.readFields(in); - edit.readFields(in); - instance = new Entry(key, edit); - } else { - int length = in.readInt(); - byte[] objectBytes = new byte[length]; - in.readFully(objectBytes); - ByteArrayInputStream bis = null; - ObjectInputStream ois = null; - try { - bis = new ByteArrayInputStream(objectBytes); - ois = new ObjectInputStream(bis); - instance = ois.readObject(); - } catch (ClassNotFoundException e) { - LOG.error("Class not found when attempting to deserialize object", e); - throw new IOException("Class not found when attempting to " + - "deserialize object", e); - } finally { - if(bis!=null) bis.close(); - if(ois!=null) ois.close(); - } - } - } - if (objectWritable != null) { // store values - objectWritable.declaredClass = declaredClass; - objectWritable.instance = instance; - } - return instance; - } - - /** - * Try to instantiate a protocol buffer of the given message class - * from the given input stream. - * - * @param protoClass the class of the generated protocol buffer - * @param dataIn the input stream to read from - * @return the instantiated Message instance - * @throws IOException if an IO problem occurs - */ - static Message tryInstantiateProtobuf( - Class protoClass, - DataInput dataIn) throws IOException { - - try { - if (dataIn instanceof InputStream) { - // We can use the built-in parseDelimitedFrom and not have to re-copy - // the data - Method parseMethod = getStaticProtobufMethod(protoClass, - "parseDelimitedFrom", InputStream.class); - return (Message)parseMethod.invoke(null, (InputStream)dataIn); - } else { - // Have to read it into a buffer first, since protobuf doesn't deal - // with the DataInput interface directly. - - // Read the size delimiter that writeDelimitedTo writes - int size = ProtoUtil.readRawVarint32(dataIn); - if (size < 0) { - throw new IOException("Invalid size: " + size); - } - - byte[] data = new byte[size]; - dataIn.readFully(data); - Method parseMethod = getStaticProtobufMethod(protoClass, - "parseFrom", byte[].class); - return (Message)parseMethod.invoke(null, data); - } - } catch (InvocationTargetException e) { - - if (e.getCause() instanceof IOException) { - throw (IOException)e.getCause(); - } else { - throw new IOException(e.getCause()); - } - } catch (IllegalAccessException iae) { - throw new AssertionError("Could not access parse method in " + - protoClass); - } - } - - static Method getStaticProtobufMethod(Class declaredClass, String method, - Class ... args) { - - try { - return declaredClass.getMethod(method, args); - } catch (Exception e) { - // This is a bug in Hadoop - protobufs should all have this static method - throw new AssertionError("Protocol buffer class " + declaredClass + - " does not have an accessible parseFrom(InputStream) method!"); - } - } - - @SuppressWarnings("unchecked") - private static Class getClassByName(Configuration conf, String className) - throws ClassNotFoundException { - if(conf != null) { - return conf.getClassByName(className); - } - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - if(cl == null) { - cl = HbaseObjectWritableFor96Migration.class.getClassLoader(); - } - return Class.forName(className, true, cl); - } - - private static void addToMap(final Class clazz, final int code) { - CLASS_TO_CODE.put(clazz, code); - CODE_TO_CLASS.put(code, clazz); - } - - public void setConf(Configuration conf) { - this.conf = conf; - } - - public Configuration getConf() { - return this.conf; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java index a9c9fe7..030d8b6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; // imports we use from yet-to-be-moved regionsever.wal import org.apache.hadoop.hbase.regionserver.wal.CompressionContext; import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -227,13 +226,11 @@ public interface WAL extends Closeable { * Utility class that lets us keep track of the edit with it's key. */ class Entry { - private WALEdit edit; - private WALKey key; + private final WALEdit edit; + private final WALKey key; public Entry() { - edit = new WALEdit(); - // we use HLogKey here instead of WALKey directly to support legacy coprocessors. - key = new HLogKey(); + this(new WALKey(), new WALEdit()); } /** @@ -243,7 +240,6 @@ public interface WAL extends Closeable { * @param key log's key */ public Entry(WALKey key, WALEdit edit) { - super(); this.key = key; this.edit = edit; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java index 8ed9bfb..abdc20c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java @@ -20,32 +20,29 @@ package org.apache.hadoop.hbase.wal; +import com.google.common.annotations.VisibleForTesting; + import java.io.IOException; -import java.util.Arrays; import java.io.InterruptedIOException; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.wal.WAL.Reader; -import org.apache.hadoop.hbase.wal.WALProvider.Writer; -import org.apache.hadoop.hbase.util.CancelableProgressable; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.LeaseNotRecoveredException; - +import org.apache.hadoop.hbase.classification.InterfaceAudience; // imports for things that haven't moved from regionserver.wal yet. import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader; -import org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; +import org.apache.hadoop.hbase.util.CancelableProgressable; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.LeaseNotRecoveredException; +import org.apache.hadoop.hbase.wal.WAL.Reader; +import org.apache.hadoop.hbase.wal.WALProvider.Writer; /** * Entry point for users of the Write Ahead Log. @@ -285,7 +282,6 @@ public class WALFactory { boolean allowCustom) throws IOException { Class lrClass = allowCustom ? logReaderClass : ProtobufLogReader.class; - try { // A wal file could be under recovery, so it may take several // tries to get it open. Instead of claiming it is corrupted, retry @@ -293,38 +289,13 @@ public class WALFactory { long startWaiting = EnvironmentEdgeManager.currentTime(); long openTimeout = timeoutMillis + startWaiting; int nbAttempt = 0; - FSDataInputStream stream = null; AbstractFSWALProvider.Reader reader = null; while (true) { try { - if (lrClass != ProtobufLogReader.class) { - // User is overriding the WAL reader, let them. - reader = lrClass.newInstance(); - reader.init(fs, path, conf, null); - return reader; - } else { - stream = fs.open(path); - // Note that zero-length file will fail to read PB magic, and attempt to create - // a non-PB reader and fail the same way existing code expects it to. If we get - // rid of the old reader entirely, we need to handle 0-size files differently from - // merely non-PB files. - byte[] magic = new byte[ProtobufLogReader.PB_WAL_MAGIC.length]; - boolean isPbWal = - (stream.read(magic) == magic.length) - && Arrays.equals(magic, ProtobufLogReader.PB_WAL_MAGIC); - reader = isPbWal ? new ProtobufLogReader() : new SequenceFileLogReader(); - reader.init(fs, path, conf, stream); - return reader; - } + reader = lrClass.newInstance(); + reader.init(fs, path, conf, null); + return reader; } catch (IOException e) { - if (stream != null) { - try { - stream.close(); - } catch (IOException exception) { - LOG.warn("Could not close AbstractFSWALProvider.Reader" + exception.getMessage()); - LOG.debug("exception details", exception); - } - } if (reader != null) { try { reader.close(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java index 4f2af38..3bf01bc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java @@ -235,15 +235,6 @@ public class WALKey implements SequenceId, Comparable { HConstants.NO_NONCE, HConstants.NO_NONCE, null, null); } - /** - * @deprecated Remove. Useless. - */ - @Deprecated // REMOVE - public WALKey(final byte[] encodedRegionName, final TableName tablename, - final NavigableMap replicationScope) { - this(encodedRegionName, tablename, System.currentTimeMillis(), replicationScope); - } - // TODO: Fix being able to pass in sequenceid. public WALKey(final byte[] encodedRegionName, final TableName tablename, final long now) { init(encodedRegionName, @@ -687,7 +678,7 @@ public class WALKey implements SequenceId, Comparable { ? UnsafeByteOperations.unsafeWrap(e.getKey()) : compressor.compress(e.getKey(), compressionContext.familyDict); builder.addScopes(FamilyScope.newBuilder() - .setFamily(family).setScopeType(ScopeType.valueOf(e.getValue()))); + .setFamily(family).setScopeType(ScopeType.forNumber(e.getValue()))); } } return builder; @@ -707,12 +698,6 @@ public class WALKey implements SequenceId, Comparable { this.tablename = TableName.valueOf(walKey.getTableName().toByteArray()); } clusterIds.clear(); - if (walKey.hasClusterId()) { - //When we are reading the older log (0.95.1 release) - //This is definitely the originating cluster - clusterIds.add(new UUID(walKey.getClusterId().getMostSigBits(), walKey.getClusterId() - .getLeastSigBits())); - } for (HBaseProtos.UUID clusterId : walKey.getClusterIdsList()) { clusterIds.add(new UUID(clusterId.getMostSigBits(), clusterId.getLeastSigBits())); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALProvider.java index 328f1b6..38c3e84 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALProvider.java @@ -81,7 +81,7 @@ public interface WALProvider { interface Writer extends Closeable { void sync() throws IOException; void append(WAL.Entry entry) throws IOException; - long getLength() throws IOException; + long getLength(); } interface AsyncWriter extends Closeable { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index 2fe9f38..1c6ab07 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -18,6 +18,10 @@ */ package org.apache.hadoop.hbase.wal; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; @@ -88,7 +92,6 @@ import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.LastSequenceId; import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.regionserver.wal.WALEditsReplaySink; @@ -119,10 +122,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; import org.apache.hadoop.io.MultipleIOException; import org.apache.hadoop.ipc.RemoteException; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - /** * This class is responsible for splitting up a bunch of regionserver commit log * files that are no longer being written to, into new files, one per region for @@ -298,13 +297,7 @@ public class WALSplitter { progress_failed = true; return false; } - try { - in = getReader(logfile, skipErrors, reporter); - } catch (CorruptedLogFileException e) { - LOG.warn("Could not get reader, corrupted log file " + logPath, e); - ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs); - isCorrupted = true; - } + in = getReader(logfile, skipErrors, reporter); if (in == null) { LOG.warn("Nothing to split in log file " + logPath); return true; @@ -388,6 +381,9 @@ public class WALSplitter { // Some tests pass in a csm of null. this.csm.getSplitLogWorkerCoordination().markCorrupted(rootDir, logfile.getPath().getName(), fs); + } else { + // for tests only + ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs); } isCorrupted = true; } catch (IOException e) { @@ -2375,8 +2371,7 @@ public class WALSplitter { for (HBaseProtos.UUID uuid : entry.getKey().getClusterIdsList()) { clusterIds.add(new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits())); } - // we use HLogKey here instead of WALKey directly to support legacy coprocessors. - key = new HLogKey(walKeyProto.getEncodedRegionName().toByteArray(), TableName.valueOf( + key = new WALKey(walKeyProto.getEncodedRegionName().toByteArray(), TableName.valueOf( walKeyProto.getTableName().toByteArray()), replaySeqId, walKeyProto.getWriteTime(), clusterIds, walKeyProto.getNonceGroup(), walKeyProto.getNonce(), null); logEntry.setFirst(key); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java index ebd6d8e..dfd1988 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java @@ -23,28 +23,23 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import org.apache.hadoop.fs.Path; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WALKey; /** - * Class for testing WALObserver coprocessor. - * - * It will monitor WAL writing and restoring, and modify passed-in WALEdit, i.e, - * ignore specified columns when writing, or add a KeyValue. On the other - * side, it checks whether the ignored column is still in WAL when Restoreed - * at region reconstruct. + * Class for testing WALObserver coprocessor. It will monitor WAL writing and restoring, and modify + * passed-in WALEdit, i.e, ignore specified columns when writing, or add a KeyValue. On the other + * side, it checks whether the ignored column is still in WAL when Restoreed at region reconstruct. */ -public class SampleRegionWALObserver extends BaseRegionObserver -implements WALObserver { +public class SampleRegionWALObserver extends BaseRegionObserver implements WALObserver { private static final Log LOG = LogFactory.getLog(SampleRegionWALObserver.class); @@ -64,12 +59,6 @@ implements WALObserver { private boolean preWALRollCalled = false; private boolean postWALRollCalled = false; - // Deprecated versions - private boolean preWALWriteDeprecatedCalled = false; - private boolean postWALWriteDeprecatedCalled = false; - private boolean preWALRestoreDeprecatedCalled = false; - private boolean postWALRestoreDeprecatedCalled = false; - /** * Set values: with a table name, a column name which will be ignored, and * a column name which will be added to WAL. @@ -88,10 +77,6 @@ implements WALObserver { postWALWriteCalled = false; preWALRestoreCalled = false; postWALRestoreCalled = false; - preWALWriteDeprecatedCalled = false; - postWALWriteDeprecatedCalled = false; - preWALRestoreDeprecatedCalled = false; - postWALRestoreDeprecatedCalled = false; preWALRollCalled = false; postWALRollCalled = false; } @@ -103,13 +88,6 @@ implements WALObserver { } @Override - public void postWALWrite(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - postWALWriteDeprecatedCalled = true; - postWALWrite(env, info, (WALKey)logKey, logEdit); - } - - @Override public boolean preWALWrite(ObserverContext env, HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { boolean bypass = false; @@ -148,13 +126,6 @@ implements WALObserver { return bypass; } - @Override - public boolean preWALWrite(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - preWALWriteDeprecatedCalled = true; - return preWALWrite(env, info, (WALKey)logKey, logEdit); - } - /** * Triggered before {@link org.apache.hadoop.hbase.regionserver.HRegion} when WAL is * Restoreed. @@ -166,13 +137,6 @@ implements WALObserver { } @Override - public void preWALRestore(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - preWALRestoreDeprecatedCalled = true; - preWALRestore(env, info, (WALKey)logKey, logEdit); - } - - @Override public void preWALRoll(ObserverContext ctx, Path oldPath, Path newPath) throws IOException { preWALRollCalled = true; @@ -194,13 +158,6 @@ implements WALObserver { postWALRestoreCalled = true; } - @Override - public void postWALRestore(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - postWALRestoreDeprecatedCalled = true; - postWALRestore(env, info, (WALKey)logKey, logEdit); - } - public boolean isPreWALWriteCalled() { return preWALWriteCalled; } @@ -221,22 +178,6 @@ implements WALObserver { return postWALRestoreCalled; } - public boolean isPreWALWriteDeprecatedCalled() { - return preWALWriteDeprecatedCalled; - } - - public boolean isPostWALWriteDeprecatedCalled() { - return postWALWriteDeprecatedCalled; - } - - public boolean isPreWALRestoreDeprecatedCalled() { - return preWALRestoreDeprecatedCalled; - } - - public boolean isPostWALRestoreDeprecatedCalled() { - return postWALRestoreDeprecatedCalled; - } - public boolean isPreWALRollCalled() { return preWALRollCalled; } @@ -244,11 +185,4 @@ implements WALObserver { public boolean isPostWALRollCalled() { return postWALRollCalled; } - - /** - * This class should trigger our legacy support since it does not directly implement the - * newer API methods. - */ - static class Legacy extends SampleRegionWALObserver { - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java index 2ea343a..d4511b9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java @@ -24,6 +24,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import com.google.common.collect.ImmutableList; + import java.io.IOException; import java.util.List; import java.util.Map; @@ -31,8 +33,6 @@ import java.util.NavigableSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -65,20 +65,16 @@ import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileReader; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WALKey; -import com.google.common.collect.ImmutableList; - /** * A sample region observer that tests the RegionObserver interface. * It works with TestRegionObserverInterface to provide the test case. */ public class SimpleRegionObserver extends BaseRegionObserver { - private static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class); final AtomicInteger ctBeforeDelete = new AtomicInteger(1); final AtomicInteger ctPreOpen = new AtomicInteger(0); @@ -130,8 +126,6 @@ public class SimpleRegionObserver extends BaseRegionObserver { final AtomicInteger ctPostReplayWALs = new AtomicInteger(0); final AtomicInteger ctPreWALRestore = new AtomicInteger(0); final AtomicInteger ctPostWALRestore = new AtomicInteger(0); - final AtomicInteger ctPreWALRestoreDeprecated = new AtomicInteger(0); - final AtomicInteger ctPostWALRestoreDeprecated = new AtomicInteger(0); final AtomicInteger ctPreSplitBeforePONR = new AtomicInteger(0); final AtomicInteger ctPreSplitAfterPONR = new AtomicInteger(0); final AtomicInteger ctPreStoreFileReaderOpen = new AtomicInteger(0); @@ -678,26 +672,12 @@ public class SimpleRegionObserver extends BaseRegionObserver { } @Override - public void preWALRestore(ObserverContext env, HRegionInfo info, - HLogKey logKey, WALEdit logEdit) throws IOException { - preWALRestore(env, info, (WALKey)logKey, logEdit); - ctPreWALRestoreDeprecated.incrementAndGet(); - } - - @Override public void postWALRestore(ObserverContext env, HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { ctPostWALRestore.incrementAndGet(); } @Override - public void postWALRestore(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - postWALRestore(env, info, (WALKey)logKey, logEdit); - ctPostWALRestoreDeprecated.incrementAndGet(); - } - - @Override public StoreFileReader preStoreFileReaderOpen(ObserverContext ctx, FileSystem fs, Path p, FSDataInputStreamWrapper in, long size, CacheConfig cacheConf, Reference r, StoreFileReader reader) throws IOException { @@ -970,22 +950,7 @@ public class SimpleRegionObserver extends BaseRegionObserver { return ctPostWALRestore.get(); } - public int getCtPreWALRestoreDeprecated() { - return ctPreWALRestoreDeprecated.get(); - } - - public int getCtPostWALRestoreDeprecated() { - return ctPostWALRestoreDeprecated.get(); - } - public boolean wasStoreFileReaderOpenCalled() { return ctPreStoreFileReaderOpen.get() > 0 && ctPostStoreFileReaderOpen.get() > 0; } - - /** - * This implementation should trigger our legacy support because it does not directly - * implement the newer API calls. - */ - public static class Legacy extends SimpleRegionObserver { - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java index bed3240..20d48bc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverInterface.java @@ -81,11 +81,10 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; -@Category({CoprocessorTests.class, MediumTests.class}) +@Category({ CoprocessorTests.class, MediumTests.class }) public class TestRegionObserverInterface { private static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class); @@ -104,8 +103,7 @@ public class TestRegionObserverInterface { Configuration conf = util.getConfiguration(); conf.setBoolean("hbase.master.distributed.log.replay", true); conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, - "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver", - "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver$Legacy"); + SimpleRegionObserver.class.getName()); util.startMiniCluster(); cluster = util.getMiniHBaseCluster(); @@ -116,17 +114,18 @@ public class TestRegionObserverInterface { util.shutdownMiniCluster(); } - @Test (timeout=300000) + @Test(timeout = 300000) public void testRegionObserver() throws IOException { TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRegionObserver"); // recreate table every time in order to reset the status of the // coprocessor. - Table table = util.createTable(tableName, new byte[][] {A, B, C}); + Table table = util.createTable(tableName, new byte[][] { A, B, C }); try { - verifyMethodResult(SimpleRegionObserver.class, new String[] { "hadPreGet", "hadPostGet", - "hadPrePut", "hadPostPut", "hadDelete", "hadPostStartRegionOperation", - "hadPostCloseRegionOperation", "hadPostBatchMutateIndispensably" }, tableName, - new Boolean[] { false, false, false, false, false, false, false, false }); + verifyMethodResult(SimpleRegionObserver.class, + new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDelete", + "hadPostStartRegionOperation", "hadPostCloseRegionOperation", + "hadPostBatchMutateIndispensably" }, + tableName, new Boolean[] { false, false, false, false, false, false, false, false }); Put put = new Put(ROW); put.addColumn(A, A, A); @@ -134,15 +133,16 @@ public class TestRegionObserverInterface { put.addColumn(C, C, C); table.put(put); - verifyMethodResult(SimpleRegionObserver.class, new String[] { "hadPreGet", "hadPostGet", - "hadPrePut", "hadPostPut", "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete", - "hadPostStartRegionOperation", "hadPostCloseRegionOperation", - "hadPostBatchMutateIndispensably" }, TEST_TABLE, new Boolean[] { false, false, true, - true, true, true, false, true, true, true }); + verifyMethodResult(SimpleRegionObserver.class, + new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadPreBatchMutate", + "hadPostBatchMutate", "hadDelete", "hadPostStartRegionOperation", + "hadPostCloseRegionOperation", "hadPostBatchMutateIndispensably" }, + TEST_TABLE, + new Boolean[] { false, false, true, true, true, true, false, true, true, true }); verifyMethodResult(SimpleRegionObserver.class, - new String[] { "getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose" }, - tableName, new Integer[] { 1, 1, 0, 0 }); + new String[] { "getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose" }, + tableName, new Integer[] { 1, 1, 0, 0 }); Get get = new Get(ROW); get.addColumn(A, A); @@ -151,9 +151,9 @@ public class TestRegionObserverInterface { table.get(get); verifyMethodResult(SimpleRegionObserver.class, - new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDelete", - "hadPrePreparedDeleteTS" }, tableName, - new Boolean[] { true, true, true, true, false, false }); + new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDelete", + "hadPrePreparedDeleteTS" }, + tableName, new Boolean[] { true, true, true, true, false, false }); Delete delete = new Delete(ROW); delete.addColumn(A, A); @@ -162,31 +162,26 @@ public class TestRegionObserverInterface { table.delete(delete); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", - "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete", "hadPrePreparedDeleteTS"}, - tableName, - new Boolean[] {true, true, true, true, true, true, true, true} - ); + new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadPreBatchMutate", + "hadPostBatchMutate", "hadDelete", "hadPrePreparedDeleteTS" }, + tableName, new Boolean[] { true, true, true, true, true, true, true, true }); } finally { util.deleteTable(tableName); table.close(); } verifyMethodResult(SimpleRegionObserver.class, - new String[] {"getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose"}, - tableName, - new Integer[] {1, 1, 1, 1}); + new String[] { "getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose" }, + tableName, new Integer[] { 1, 1, 1, 1 }); } - @Test (timeout=300000) + @Test(timeout = 300000) public void testRowMutation() throws IOException { TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRowMutation"); Table table = util.createTable(tableName, new byte[][] { A, B, C }); try { verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", - "hadDeleted"}, - tableName, - new Boolean[] {false, false, false, false, false}); + new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDeleted" }, + tableName, new Boolean[] { false, false, false, false, false }); Put put = new Put(ROW); put.addColumn(A, A, A); put.addColumn(B, B, B); @@ -203,18 +198,15 @@ public class TestRegionObserverInterface { table.mutateRow(arm); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", - "hadDeleted"}, - tableName, - new Boolean[] {false, false, true, true, true} - ); + new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDeleted" }, + tableName, new Boolean[] { false, false, true, true, true }); } finally { util.deleteTable(tableName); table.close(); } } - @Test (timeout=300000) + @Test(timeout = 300000) public void testIncrementHook() throws IOException { TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testIncrementHook"); Table table = util.createTable(tableName, new byte[][] { A, B, C }); @@ -223,80 +215,68 @@ public class TestRegionObserverInterface { inc.addColumn(A, A, 1); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock"}, - tableName, - new Boolean[] {false, false, false} - ); + new String[] { "hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock" }, + tableName, new Boolean[] { false, false, false }); table.increment(inc); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock"}, - tableName, - new Boolean[] {true, true, true} - ); + new String[] { "hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock" }, + tableName, new Boolean[] { true, true, true }); } finally { util.deleteTable(tableName); table.close(); } } - @Test (timeout=300000) + @Test(timeout = 300000) public void testCheckAndPutHooks() throws IOException { - TableName tableName = - TableName.valueOf(TEST_TABLE.getNameAsString() + ".testCheckAndPutHooks"); - try (Table table = util.createTable(tableName, new byte[][] {A, B, C})) { + TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testCheckAndPutHooks"); + try (Table table = util.createTable(tableName, new byte[][] { A, B, C })) { Put p = new Put(Bytes.toBytes(0)); p.addColumn(A, A, A); table.put(p); p = new Put(Bytes.toBytes(0)); p.addColumn(A, A, A); verifyMethodResult(SimpleRegionObserver.class, - new String[] { "hadPreCheckAndPut", "hadPreCheckAndPutAfterRowLock", - "hadPostCheckAndPut" }, tableName, new Boolean[] { false, false, false }); + new String[] { "hadPreCheckAndPut", "hadPreCheckAndPutAfterRowLock", "hadPostCheckAndPut" }, + tableName, new Boolean[] { false, false, false }); table.checkAndPut(Bytes.toBytes(0), A, A, A, p); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreCheckAndPut", - "hadPreCheckAndPutAfterRowLock", "hadPostCheckAndPut"}, - tableName, - new Boolean[] {true, true, true} - ); + new String[] { "hadPreCheckAndPut", "hadPreCheckAndPutAfterRowLock", "hadPostCheckAndPut" }, + tableName, new Boolean[] { true, true, true }); } finally { util.deleteTable(tableName); } } - @Test (timeout=300000) + @Test(timeout = 300000) public void testCheckAndDeleteHooks() throws IOException { TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testCheckAndDeleteHooks"); - Table table = util.createTable(tableName, new byte[][] {A, B, C}); + Table table = util.createTable(tableName, new byte[][] { A, B, C }); try { Put p = new Put(Bytes.toBytes(0)); p.addColumn(A, A, A); table.put(p); Delete d = new Delete(Bytes.toBytes(0)); table.delete(d); - verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreCheckAndDelete", - "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete"}, - tableName, - new Boolean[] {false, false, false} - ); + verifyMethodResult( + SimpleRegionObserver.class, new String[] { "hadPreCheckAndDelete", + "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete" }, + tableName, new Boolean[] { false, false, false }); table.checkAndDelete(Bytes.toBytes(0), A, A, A, d); - verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreCheckAndDelete", - "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete"}, - tableName, - new Boolean[] {true, true, true} - ); + verifyMethodResult( + SimpleRegionObserver.class, new String[] { "hadPreCheckAndDelete", + "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete" }, + tableName, new Boolean[] { true, true, true }); } finally { util.deleteTable(tableName); table.close(); } } - @Test (timeout=300000) + @Test(timeout = 300000) public void testAppendHook() throws IOException { TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testAppendHook"); Table table = util.createTable(tableName, new byte[][] { A, B, C }); @@ -305,35 +285,30 @@ public class TestRegionObserverInterface { app.add(A, A, A); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock"}, - tableName, - new Boolean[] {false, false, false} - ); + new String[] { "hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock" }, tableName, + new Boolean[] { false, false, false }); table.append(app); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock"}, - tableName, - new Boolean[] {true, true, true} - ); + new String[] { "hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock" }, tableName, + new Boolean[] { true, true, true }); } finally { util.deleteTable(tableName); table.close(); } } - @Test (timeout=300000) + @Test(timeout = 300000) // HBase-3583 public void testHBase3583() throws IOException { - TableName tableName = - TableName.valueOf("testHBase3583"); - util.createTable(tableName, new byte[][] {A, B, C}); + TableName tableName = TableName.valueOf("testHBase3583"); + util.createTable(tableName, new byte[][] { A, B, C }); util.waitUntilAllRegionsAssigned(tableName); verifyMethodResult(SimpleRegionObserver.class, - new String[] { "hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled" }, - tableName, new Boolean[] { false, false, false, false }); + new String[] { "hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled" }, + tableName, new Boolean[] { false, false, false, false }); Table table = util.getConnection().getTable(tableName); Put put = new Put(ROW); @@ -347,11 +322,8 @@ public class TestRegionObserverInterface { // verify that scannerNext and scannerClose upcalls won't be invoked // when we perform get(). verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled", - "wasScannerCloseCalled"}, - tableName, - new Boolean[] {true, true, false, false} - ); + new String[] { "hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled" }, + tableName, new Boolean[] { true, true, false, false }); Scan s = new Scan(); ResultScanner scanner = table.getScanner(s); @@ -364,15 +336,13 @@ public class TestRegionObserverInterface { // now scanner hooks should be invoked. verifyMethodResult(SimpleRegionObserver.class, - new String[] {"wasScannerNextCalled", "wasScannerCloseCalled"}, - tableName, - new Boolean[] {true, true} - ); + new String[] { "wasScannerNextCalled", "wasScannerCloseCalled" }, tableName, + new Boolean[] { true, true }); util.deleteTable(tableName); table.close(); } - @Test (timeout=300000) + @Test(timeout = 300000) public void testHBASE14489() throws IOException { TableName tableName = TableName.valueOf("testHBASE14489"); Table table = util.createTable(tableName, new byte[][] { A }); @@ -396,18 +366,15 @@ public class TestRegionObserverInterface { } - @Test (timeout=300000) + @Test(timeout = 300000) // HBase-3758 public void testHBase3758() throws IOException { - TableName tableName = - TableName.valueOf("testHBase3758"); - util.createTable(tableName, new byte[][] {A, B, C}); + TableName tableName = TableName.valueOf("testHBase3758"); + util.createTable(tableName, new byte[][] { A, B, C }); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadDeleted", "wasScannerOpenCalled"}, - tableName, - new Boolean[] {false, false} - ); + new String[] { "hadDeleted", "wasScannerOpenCalled" }, tableName, + new Boolean[] { false, false }); Table table = util.getConnection().getTable(tableName); Put put = new Put(ROW); @@ -418,10 +385,8 @@ public class TestRegionObserverInterface { table.delete(delete); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadDeleted", "wasScannerOpenCalled"}, - tableName, - new Boolean[] {true, false} - ); + new String[] { "hadDeleted", "wasScannerOpenCalled" }, tableName, + new Boolean[] { true, false }); Scan s = new Scan(); ResultScanner scanner = table.getScanner(s); @@ -433,11 +398,8 @@ public class TestRegionObserverInterface { } // now scanner hooks should be invoked. - verifyMethodResult(SimpleRegionObserver.class, - new String[] {"wasScannerOpenCalled"}, - tableName, - new Boolean[] {true} - ); + verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerOpenCalled" }, + tableName, new Boolean[] { true }); util.deleteTable(tableName); table.close(); } @@ -448,8 +410,8 @@ public class TestRegionObserverInterface { long lastFlush; @Override - public InternalScanner preCompact(ObserverContext e, - Store store, final InternalScanner scanner, final ScanType scanType) { + public InternalScanner preCompact(ObserverContext e, Store store, + final InternalScanner scanner, final ScanType scanType) { return new InternalScanner() { @Override public boolean next(List results) throws IOException { @@ -457,8 +419,7 @@ public class TestRegionObserverInterface { } @Override - public boolean next(List results, ScannerContext scannerContext) - throws IOException { + public boolean next(List results, ScannerContext scannerContext) throws IOException { List internalResults = new ArrayList(); boolean hasMore; do { @@ -488,8 +449,8 @@ public class TestRegionObserverInterface { } @Override - public void postCompact(ObserverContext e, - Store store, StoreFile resultFile) { + public void postCompact(ObserverContext e, Store store, + StoreFile resultFile) { lastCompaction = EnvironmentEdgeManager.currentTime(); } @@ -498,14 +459,15 @@ public class TestRegionObserverInterface { lastFlush = EnvironmentEdgeManager.currentTime(); } } + /** * Tests overriding compaction handling via coprocessor hooks * @throws Exception */ - @Test (timeout=300000) + @Test(timeout = 300000) public void testCompactionOverride() throws Exception { TableName compactTable = TableName.valueOf("TestCompactionOverride"); - Admin admin = util.getHBaseAdmin(); + Admin admin = util.getAdmin(); if (admin.tableExists(compactTable)) { admin.disableTable(compactTable); admin.deleteTable(compactTable); @@ -517,7 +479,7 @@ public class TestRegionObserverInterface { admin.createTable(htd); Table table = util.getConnection().getTable(compactTable); - for (long i=1; i<=10; i++) { + for (long i = 1; i <= 10; i++) { byte[] iBytes = Bytes.toBytes(i); Put put = new Put(iBytes); put.setDurability(Durability.SKIP_WAL); @@ -526,16 +488,16 @@ public class TestRegionObserverInterface { } HRegion firstRegion = cluster.getRegions(compactTable).get(0); - Coprocessor cp = firstRegion.getCoprocessorHost().findCoprocessor( - EvenOnlyCompactor.class.getName()); + Coprocessor cp = + firstRegion.getCoprocessorHost().findCoprocessor(EvenOnlyCompactor.class.getName()); assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp); - EvenOnlyCompactor compactor = (EvenOnlyCompactor)cp; + EvenOnlyCompactor compactor = (EvenOnlyCompactor) cp; // force a compaction long ts = System.currentTimeMillis(); admin.flush(compactTable); // wait for flush - for (int i=0; i<10; i++) { + for (int i = 0; i < 10; i++) { if (compactor.lastFlush >= ts) { break; } @@ -547,25 +509,25 @@ public class TestRegionObserverInterface { ts = compactor.lastFlush; admin.majorCompact(compactTable); // wait for compaction - for (int i=0; i<30; i++) { + for (int i = 0; i < 30; i++) { if (compactor.lastCompaction >= ts) { break; } Thread.sleep(1000); } - LOG.debug("Last compaction was at "+compactor.lastCompaction); + LOG.debug("Last compaction was at " + compactor.lastCompaction); assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts); // only even rows should remain ResultScanner scanner = table.getScanner(new Scan()); try { - for (long i=2; i<=10; i+=2) { + for (long i = 2; i <= 10; i += 2) { Result r = scanner.next(); assertNotNull(r); assertFalse(r.isEmpty()); byte[] iBytes = Bytes.toBytes(i); - assertArrayEquals("Row should be "+i, r.getRow(), iBytes); - assertArrayEquals("Value should be "+i, r.getValue(A, A), iBytes); + assertArrayEquals("Row should be " + i, r.getRow(), iBytes); + assertArrayEquals("Value should be " + i, r.getValue(A, A), iBytes); } } finally { scanner.close(); @@ -573,18 +535,16 @@ public class TestRegionObserverInterface { table.close(); } - @Test (timeout=300000) + @Test(timeout = 300000) public void bulkLoadHFileTest() throws Exception { - String testName = TestRegionObserverInterface.class.getName()+".bulkLoadHFileTest"; + String testName = TestRegionObserverInterface.class.getName() + ".bulkLoadHFileTest"; TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".bulkLoadHFileTest"); Configuration conf = util.getConfiguration(); - Table table = util.createTable(tableName, new byte[][] {A, B, C}); + Table table = util.createTable(tableName, new byte[][] { A, B, C }); try (RegionLocator locator = util.getConnection().getRegionLocator(tableName)) { verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"}, - tableName, - new Boolean[] {false, false} - ); + new String[] { "hadPreBulkLoadHFile", "hadPostBulkLoadHFile" }, tableName, + new Boolean[] { false, false }); FileSystem fs = util.getTestFileSystem(); final Path dir = util.getDataTestDirOnTestFS(testName).makeQualified(fs); @@ -593,32 +553,30 @@ public class TestRegionObserverInterface { createHFile(util.getConfiguration(), fs, new Path(familyDir, Bytes.toString(A)), A, A); // Bulk load - new LoadIncrementalHFiles(conf).doBulkLoad(dir, util.getHBaseAdmin(), table, locator); + new LoadIncrementalHFiles(conf).doBulkLoad(dir, util.getAdmin(), table, locator); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"}, - tableName, - new Boolean[] {true, true} - ); + new String[] { "hadPreBulkLoadHFile", "hadPostBulkLoadHFile" }, tableName, + new Boolean[] { true, true }); } finally { util.deleteTable(tableName); table.close(); } } - @Test (timeout=300000) + @Test(timeout = 300000) public void testRecovery() throws Exception { - LOG.info(TestRegionObserverInterface.class.getName() +".testRecovery"); + LOG.info(TestRegionObserverInterface.class.getName() + ".testRecovery"); TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRecovery"); - Table table = util.createTable(tableName, new byte[][] {A, B, C}); + Table table = util.createTable(tableName, new byte[][] { A, B, C }); try (RegionLocator locator = util.getConnection().getRegionLocator(tableName)) { JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer(); ServerName sn2 = rs1.getRegionServer().getServerName(); String regEN = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName(); - util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes()); - while (!sn2.equals(locator.getAllRegionLocations().get(0).getServerName())){ + util.getAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes()); + while (!sn2.equals(locator.getAllRegionLocations().get(0).getServerName())) { Thread.sleep(100); } @@ -632,18 +590,14 @@ public class TestRegionObserverInterface { table.put(put); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", - "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"}, - tableName, - new Boolean[] {false, false, true, true, true, true, false} - ); + new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadPreBatchMutate", + "hadPostBatchMutate", "hadDelete" }, + tableName, new Boolean[] { false, false, true, true, true, true, false }); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore", - "getCtPostWALRestore", "getCtPrePut", "getCtPostPut", - "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"}, - tableName, - new Integer[] {0, 0, 0, 0, 2, 2, 0, 0}); + new String[] { "getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore", + "getCtPostWALRestore", "getCtPrePut", "getCtPostPut" }, + tableName, new Integer[] { 0, 0, 0, 0, 2, 2 }); cluster.killRegionServer(rs1.getRegionServer().getServerName()); Threads.sleep(1000); // Let the kill soak in. @@ -651,75 +605,16 @@ public class TestRegionObserverInterface { LOG.info("All regions assigned"); verifyMethodResult(SimpleRegionObserver.class, - new String[] {"getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore", - "getCtPostWALRestore", "getCtPrePut", "getCtPostPut", - "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"}, - tableName, - new Integer[]{1, 1, 2, 2, 0, 0, 0, 0}); + new String[] { "getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore", + "getCtPostWALRestore", "getCtPrePut", "getCtPostPut" }, + tableName, new Integer[] { 1, 1, 2, 2, 0, 0 }); } finally { util.deleteTable(tableName); table.close(); } } - @Test (timeout=300000) - public void testLegacyRecovery() throws Exception { - LOG.info(TestRegionObserverInterface.class.getName() +".testLegacyRecovery"); - TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testLegacyRecovery"); - Table table = util.createTable(tableName, new byte[][] {A, B, C}); - try { - try (RegionLocator locator = util.getConnection().getRegionLocator(tableName)) { - JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer(); - ServerName sn2 = rs1.getRegionServer().getServerName(); - String regEN = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName(); - - util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes()); - while (!sn2.equals(locator.getAllRegionLocations().get(0).getServerName())) { - Thread.sleep(100); - } - - Put put = new Put(ROW); - put.addColumn(A, A, A); - put.addColumn(B, B, B); - put.addColumn(C, C, C); - table.put(put); - - // put two times - table.put(put); - - verifyMethodResult(SimpleRegionObserver.Legacy.class, - new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", - "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"}, - tableName, - new Boolean[] {false, false, true, true, true, true, false} - ); - - verifyMethodResult(SimpleRegionObserver.Legacy.class, - new String[] {"getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore", - "getCtPostWALRestore", "getCtPrePut", "getCtPostPut", - "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"}, - tableName, - new Integer[] {0, 0, 0, 0, 2, 2, 0, 0}); - - cluster.killRegionServer(rs1.getRegionServer().getServerName()); - Threads.sleep(1000); // Let the kill soak in. - util.waitUntilAllRegionsAssigned(tableName); - LOG.info("All regions assigned"); - - verifyMethodResult(SimpleRegionObserver.Legacy.class, - new String[] {"getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore", - "getCtPostWALRestore", "getCtPrePut", "getCtPostPut", - "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"}, - tableName, - new Integer[]{1, 1, 2, 2, 0, 0, 2, 2}); - } - } finally { - util.deleteTable(tableName); - table.close(); - } - } - - @Test (timeout=300000) + @Test(timeout = 300000) public void testPreWALRestoreSkip() throws Exception { LOG.info(TestRegionObserverInterface.class.getName() + ".testPreWALRestoreSkip"); TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED); @@ -730,7 +625,7 @@ public class TestRegionObserverInterface { ServerName sn2 = rs1.getRegionServer().getServerName(); String regEN = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName(); - util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes()); + util.getAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes()); while (!sn2.equals(locator.getAllRegionLocations().get(0).getServerName())) { Thread.sleep(100); } @@ -746,10 +641,9 @@ public class TestRegionObserverInterface { util.waitUntilAllRegionsAssigned(tableName); } - verifyMethodResult(SimpleRegionObserver.class, new String[] { "getCtPreWALRestore", - "getCtPostWALRestore", "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"}, - tableName, - new Integer[] {0, 0, 0, 0}); + verifyMethodResult(SimpleRegionObserver.class, + new String[] { "getCtPreWALRestore", "getCtPostWALRestore", }, tableName, + new Integer[] { 0, 0 }); util.deleteTable(tableName); table.close(); @@ -757,27 +651,28 @@ public class TestRegionObserverInterface { // check each region whether the coprocessor upcalls are called or not. private void verifyMethodResult(Class c, String methodName[], TableName tableName, - Object value[]) throws IOException { + Object value[]) throws IOException { try { for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) { - if (!t.isAlive() || t.getRegionServer().isAborted() || t.getRegionServer().isStopping()){ + if (!t.isAlive() || t.getRegionServer().isAborted() || t.getRegionServer().isStopping()) { continue; } - for (HRegionInfo r : ProtobufUtil.getOnlineRegions(t.getRegionServer().getRSRpcServices())) { + for (HRegionInfo r : ProtobufUtil + .getOnlineRegions(t.getRegionServer().getRSRpcServices())) { if (!r.getTable().equals(tableName)) { continue; } - RegionCoprocessorHost cph = t.getRegionServer().getOnlineRegion(r.getRegionName()). - getCoprocessorHost(); + RegionCoprocessorHost cph = + t.getRegionServer().getOnlineRegion(r.getRegionName()).getCoprocessorHost(); Coprocessor cp = cph.findCoprocessor(c.getName()); assertNotNull(cp); for (int i = 0; i < methodName.length; ++i) { Method m = c.getMethod(methodName[i]); Object o = m.invoke(cp); - assertTrue("Result of " + c.getName() + "." + methodName[i] - + " is expected to be " + value[i].toString() - + ", while we get " + o.toString(), o.equals(value[i])); + assertTrue("Result of " + c.getName() + "." + methodName[i] + " is expected to be " + + value[i].toString() + ", while we get " + o.toString(), + o.equals(value[i])); } } } @@ -786,19 +681,16 @@ public class TestRegionObserverInterface { } } - private static void createHFile( - Configuration conf, - FileSystem fs, Path path, - byte[] family, byte[] qualifier) throws IOException { + private static void createHFile(Configuration conf, FileSystem fs, Path path, byte[] family, + byte[] qualifier) throws IOException { HFileContext context = new HFileContextBuilder().build(); - HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)) - .withPath(fs, path) - .withFileContext(context) - .create(); + HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path) + .withFileContext(context).create(); long now = System.currentTimeMillis(); try { - for (int i =1;i<=9;i++) { - KeyValue kv = new KeyValue(Bytes.toBytes(i+""), family, qualifier, now, Bytes.toBytes(i+"")); + for (int i = 1; i <= 9; i++) { + KeyValue kv = + new KeyValue(Bytes.toBytes(i + ""), family, qualifier, now, Bytes.toBytes(i + "")); writer.append(kv); } } finally { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java index 3c591f8..6eca7f0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java @@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.security.User; @@ -110,7 +109,7 @@ public class TestWALObserver { public static void setupBeforeClass() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, - SampleRegionWALObserver.class.getName(), SampleRegionWALObserver.Legacy.class.getName()); + SampleRegionWALObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, SampleRegionWALObserver.class.getName()); conf.setInt("dfs.client.block.recovery.retries", 2); @@ -169,17 +168,6 @@ public class TestWALObserver { verifyWritesSeen(log, getCoprocessor(log, SampleRegionWALObserver.class), false); } - /** - * Test WAL write behavior with WALObserver. The coprocessor monitors a - * WALEdit written to WAL, and ignore, modify, and add KeyValue's for the - * WALEdit. - */ - @Test - public void testLegacyWALObserverWriteToWAL() throws Exception { - final WAL log = wals.getWAL(UNSPECIFIED_REGION, null); - verifyWritesSeen(log, getCoprocessor(log, SampleRegionWALObserver.Legacy.class), true); - } - private void verifyWritesSeen(final WAL log, final SampleRegionWALObserver cp, final boolean seesLegacy) throws Exception { HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE)); @@ -202,8 +190,6 @@ public class TestWALObserver { assertFalse(cp.isPreWALWriteCalled()); assertFalse(cp.isPostWALWriteCalled()); - assertFalse(cp.isPreWALWriteDeprecatedCalled()); - assertFalse(cp.isPostWALWriteDeprecatedCalled()); // TEST_FAMILY[2] is not in the put, however it shall be added by the tested // coprocessor. @@ -241,7 +227,7 @@ public class TestWALObserver { long now = EnvironmentEdgeManager.currentTime(); // we use HLogKey here instead of WALKey directly to support legacy coprocessors. long txid = log.append(hri, - new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, scopes), edit, true); + new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, scopes), edit, true); log.sync(txid); // the edit shall have been change now by the coprocessor. @@ -267,88 +253,6 @@ public class TestWALObserver { assertTrue(cp.isPreWALWriteCalled()); assertTrue(cp.isPostWALWriteCalled()); - assertEquals(seesLegacy, cp.isPreWALWriteDeprecatedCalled()); - assertEquals(seesLegacy, cp.isPostWALWriteDeprecatedCalled()); - } - - @Test - public void testNonLegacyWALKeysDoNotExplode() throws Exception { - TableName tableName = TableName.valueOf(TEST_TABLE); - final HTableDescriptor htd = createBasic3FamilyHTD(Bytes - .toString(TEST_TABLE)); - final HRegionInfo hri = new HRegionInfo(tableName, null, null); - MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); - - fs.mkdirs(new Path(FSUtils.getTableDir(hbaseRootDir, tableName), hri.getEncodedName())); - - final Configuration newConf = HBaseConfiguration.create(this.conf); - - final WAL wal = wals.getWAL(UNSPECIFIED_REGION, null); - final SampleRegionWALObserver newApi = getCoprocessor(wal, SampleRegionWALObserver.class); - newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null); - final SampleRegionWALObserver oldApi = getCoprocessor(wal, - SampleRegionWALObserver.Legacy.class); - oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null); - - LOG.debug("ensuring wal entries haven't happened before we start"); - assertFalse(newApi.isPreWALWriteCalled()); - assertFalse(newApi.isPostWALWriteCalled()); - assertFalse(newApi.isPreWALWriteDeprecatedCalled()); - assertFalse(newApi.isPostWALWriteDeprecatedCalled()); - assertFalse(oldApi.isPreWALWriteCalled()); - assertFalse(oldApi.isPostWALWriteCalled()); - assertFalse(oldApi.isPreWALWriteDeprecatedCalled()); - assertFalse(oldApi.isPostWALWriteDeprecatedCalled()); - - LOG.debug("writing to WAL with non-legacy keys."); - NavigableMap scopes = new TreeMap( - Bytes.BYTES_COMPARATOR); - for (HColumnDescriptor hcd : htd.getFamilies()) { - scopes.put(hcd.getName(), 0); - } - final int countPerFamily = 5; - for (HColumnDescriptor hcd : htd.getFamilies()) { - addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, - EnvironmentEdgeManager.getDelegate(), wal, scopes, mvcc); - } - - LOG.debug("Verify that only the non-legacy CP saw edits."); - assertTrue(newApi.isPreWALWriteCalled()); - assertTrue(newApi.isPostWALWriteCalled()); - assertFalse(newApi.isPreWALWriteDeprecatedCalled()); - assertFalse(newApi.isPostWALWriteDeprecatedCalled()); - // wish we could test that the log message happened :/ - assertFalse(oldApi.isPreWALWriteCalled()); - assertFalse(oldApi.isPostWALWriteCalled()); - assertFalse(oldApi.isPreWALWriteDeprecatedCalled()); - assertFalse(oldApi.isPostWALWriteDeprecatedCalled()); - - LOG.debug("reseting cp state."); - newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null); - oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null); - - LOG.debug("write a log edit that supports legacy cps."); - final long now = EnvironmentEdgeManager.currentTime(); - final WALKey legacyKey = new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), now); - final WALEdit edit = new WALEdit(); - final byte[] nonce = Bytes.toBytes("1772"); - edit.add(new KeyValue(TEST_ROW, TEST_FAMILY[0], nonce, now, nonce)); - final long txid = wal.append(hri, legacyKey, edit, true); - wal.sync(txid); - - LOG.debug("Make sure legacy cps can see supported edits after having been skipped."); - assertTrue("non-legacy WALObserver didn't see pre-write.", newApi.isPreWALWriteCalled()); - assertTrue("non-legacy WALObserver didn't see post-write.", newApi.isPostWALWriteCalled()); - assertFalse("non-legacy WALObserver shouldn't have seen legacy pre-write.", - newApi.isPreWALWriteDeprecatedCalled()); - assertFalse("non-legacy WALObserver shouldn't have seen legacy post-write.", - newApi.isPostWALWriteDeprecatedCalled()); - assertTrue("legacy WALObserver didn't see pre-write.", oldApi.isPreWALWriteCalled()); - assertTrue("legacy WALObserver didn't see post-write.", oldApi.isPostWALWriteCalled()); - assertTrue("legacy WALObserver didn't see legacy pre-write.", - oldApi.isPreWALWriteDeprecatedCalled()); - assertTrue("legacy WALObserver didn't see legacy post-write.", - oldApi.isPostWALWriteDeprecatedCalled()); } /** @@ -431,8 +335,8 @@ public class TestWALObserver { User user = HBaseTestingUtility.getDifferentUser(newConf, ".replay.wal.secondtime"); - user.runAs(new PrivilegedExceptionAction() { - public Object run() throws Exception { + user.runAs(new PrivilegedExceptionAction() { + public Void run() throws Exception { Path p = runWALSplit(newConf); LOG.info("WALSplit path == " + p); FileSystem newFS = FileSystem.get(newConf); @@ -450,8 +354,6 @@ public class TestWALObserver { assertNotNull(cp2); assertTrue(cp2.isPreWALRestoreCalled()); assertTrue(cp2.isPostWALRestoreCalled()); - assertFalse(cp2.isPreWALRestoreDeprecatedCalled()); - assertFalse(cp2.isPostWALRestoreDeprecatedCalled()); region.close(); wals2.close(); return null; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java deleted file mode 100644 index 752faa6..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.mapreduce; - -import org.apache.hadoop.hbase.mapreduce.WALInputFormat.WALRecordReader; - -import java.util.NavigableMap; - -import org.apache.hadoop.hbase.mapreduce.HLogInputFormat.HLogKeyRecordReader; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.experimental.categories.Category; - -/** - * JUnit tests for the record reader in HLogInputFormat - */ -@Category({MapReduceTests.class, MediumTests.class}) -public class TestHLogRecordReader extends TestWALRecordReader { - - @Override - protected WALKey getWalKey(final long time, NavigableMap scopes) { - return new HLogKey(info.getEncodedNameAsBytes(), tableName, time, mvcc, scopes); - } - - @Override - protected WALRecordReader getReader() { - return new HLogKeyRecordReader(); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index e34c9cd..5078038 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -89,7 +89,6 @@ import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -101,9 +100,9 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; -import org.apache.hadoop.hbase.wal.FSHLogProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; +import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -1308,8 +1307,8 @@ public class TestDistributedLogSplitting { WALEdit e = new WALEdit(); value++; e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); - wal.append(curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(), tableName, - System.currentTimeMillis(), null), e, true); + wal.append(curRegionInfo, new WALKey(curRegionInfo.getEncodedNameAsBytes(), tableName, + System.currentTimeMillis()), e, true); } wal.sync(); wal.shutdown(); @@ -1403,7 +1402,7 @@ public class TestDistributedLogSplitting { WALEdit e = new WALEdit(); value++; e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); - wal.append(curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(), + wal.append(curRegionInfo, new WALKey(curRegionInfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis()), e, true); } wal.sync(); @@ -1617,7 +1616,7 @@ public class TestDistributedLogSplitting { byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i)); e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value)); log.append(curRegionInfo, - new HLogKey(curRegionInfo.getEncodedNameAsBytes(), fullTName, + new WALKey(curRegionInfo.getEncodedNameAsBytes(), fullTName, System.currentTimeMillis()), e, true); if (0 == i % syncEvery) { log.sync(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java index ad88cfe..73fb9cf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java @@ -150,7 +150,7 @@ public class TestFailedAppendAndSync { } @Override - public long getLength() throws IOException { + public long getLength() { return w.getLength(); } }; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 4c8b1de..bd1ec5c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -43,6 +43,10 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + import java.io.IOException; import java.io.InterruptedIOException; import java.security.PrivilegedExceptionAction; @@ -127,19 +131,11 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; import org.apache.hadoop.hbase.regionserver.Region.RowLock; import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem; import org.apache.hadoop.hbase.regionserver.handler.FinishRegionRecoveringHandler; import org.apache.hadoop.hbase.regionserver.wal.FSHLog; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; @@ -147,6 +143,13 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.regionserver.wal.WALUtil; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; @@ -181,10 +184,6 @@ import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - /** * Basic stand-alone testing of HRegion. No clusters! * @@ -695,7 +694,7 @@ public class TestHRegion { WALEdit edit = new WALEdit(); edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); - writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time, + writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); @@ -746,7 +745,7 @@ public class TestHRegion { WALEdit edit = new WALEdit(); edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); - writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time, + writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); @@ -849,7 +848,7 @@ public class TestHRegion { edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); } - writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time, + writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); } @@ -946,7 +945,7 @@ public class TestHRegion { long time = System.nanoTime(); - writer.append(new WAL.Entry(new HLogKey(regionName, tableName, 10, time, + writer.append(new WAL.Entry(new WALKey(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor))); writer.close(); @@ -1187,7 +1186,7 @@ public class TestHRegion { } @Override - public long getLength() throws IOException { + public long getLength() { return w.getLength(); } }; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java index 31f9a42..188016c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java @@ -190,7 +190,7 @@ public class TestWALLockup { } @Override - public long getLength() throws IOException { + public long getLength() { return w.getLength(); } }; @@ -228,7 +228,7 @@ public class TestWALLockup { Put put = new Put(bytes); put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes); WALKey key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), - scopes); + System.currentTimeMillis(), scopes); WALEdit edit = new WALEdit(); CellScanner CellScanner = put.cellScanner(); assertTrue(CellScanner.advance()); @@ -349,7 +349,7 @@ public class TestWALLockup { } @Override - public long getLength() throws IOException { + public long getLength() { return w.getLength(); } }; @@ -403,8 +403,8 @@ public class TestWALLockup { try { Put put = new Put(bytes); put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes); - WALKey key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), - htd.getTableName(), scopes); + WALKey key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), + System.currentTimeMillis(), scopes); WALEdit edit = new WALEdit(); CellScanner CellScanner = put.cellScanner(); assertTrue(CellScanner.advance()); @@ -435,8 +435,8 @@ public class TestWALLockup { // make RingBufferEventHandler sleep 1s, so the following sync // endOfBatch=false - key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), - TableName.valueOf("sleep"), scopes); + key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), TableName.valueOf("sleep"), + System.currentTimeMillis(), scopes); dodgyWAL2.append(region.getRegionInfo(), key, edit, true); Thread t = new Thread("Sync") { @@ -460,7 +460,7 @@ public class TestWALLockup { } // make append throw DamagedWALException key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), - TableName.valueOf("DamagedWALException"), scopes); + TableName.valueOf("DamagedWALException"), System.currentTimeMillis(), scopes); dodgyWAL2.append(region.getRegionInfo(), key, edit, true); while (latch.getCount() > 0) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java new file mode 100644 index 0000000..c654c16 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.Queue; + +import org.apache.hadoop.hbase.wal.WAL.Entry; +import org.apache.hadoop.hbase.wal.WALKey; + +public class FaultyProtobufLogReader extends ProtobufLogReader { + + // public until class relocates to o.a.h.h.wal + public enum FailureType { + BEGINNING, MIDDLE, END, NONE + } + + Queue nextQueue = new LinkedList(); + int numberOfFileEntries = 0; + + FailureType getFailureType() { + return FailureType.valueOf(conf.get("faultyprotobuflogreader.failuretype", "NONE")); + } + + @Override + public Entry next(Entry reuse) throws IOException { + if (nextQueue.isEmpty()) { // Read the whole thing at once and fake reading + boolean b; + do { + Entry e = new Entry(new WALKey(), new WALEdit()); + if (compressionContext != null) { + e.setCompressionContext(compressionContext); + } + b = readNext(e); + nextQueue.offer(e); + numberOfFileEntries++; + } while (b); + } + + if (nextQueue.size() == this.numberOfFileEntries && getFailureType() == FailureType.BEGINNING) { + throw new IOException("fake Exception"); + } else if (nextQueue.size() == this.numberOfFileEntries / 2 + && getFailureType() == FailureType.MIDDLE) { + throw new IOException("fake Exception"); + } else if (nextQueue.size() == 1 && getFailureType() == FailureType.END) { + throw new IOException("fake Exception"); + } + + if (nextQueue.peek() != null) { + edit++; + } + + Entry e = nextQueue.poll(); + + if (e.getEdit().isEmpty()) { + return null; + } + return e; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java deleted file mode 100644 index a0e4490..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultySequenceFileLogReader.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver.wal; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.Queue; - -import org.apache.hadoop.hbase.wal.WAL.Entry; - -public class FaultySequenceFileLogReader extends SequenceFileLogReader { - - // public until class relocates to o.a.h.h.wal - public enum FailureType { - BEGINNING, MIDDLE, END, NONE - } - - Queue nextQueue = new LinkedList(); - int numberOfFileEntries = 0; - - FailureType getFailureType() { - return FailureType.valueOf(conf.get("faultysequencefilelogreader.failuretype", "NONE")); - } - - @Override - public Entry next(Entry reuse) throws IOException { - this.entryStart = this.getPosition(); - boolean b = true; - - if (nextQueue.isEmpty()) { // Read the whole thing at once and fake reading - while (b == true) { - Entry e = new Entry(new HLogKey(), new WALEdit()); - if (compressionContext != null) { - e.setCompressionContext(compressionContext); - } - b = readNext(e); - nextQueue.offer(e); - numberOfFileEntries++; - } - } - - if (nextQueue.size() == this.numberOfFileEntries - && getFailureType() == FailureType.BEGINNING) { - throw this.addFileInfoToException(new IOException("fake Exception")); - } else if (nextQueue.size() == this.numberOfFileEntries / 2 - && getFailureType() == FailureType.MIDDLE) { - throw this.addFileInfoToException(new IOException("fake Exception")); - } else if (nextQueue.size() == 1 && getFailureType() == FailureType.END) { - throw this.addFileInfoToException(new IOException("fake Exception")); - } - - if (nextQueue.peek() != null) { - edit++; - } - - Entry e = nextQueue.poll(); - - if (e.getEdit().isEmpty()) { - return null; - } - return e; - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java deleted file mode 100644 index 101758e..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java +++ /dev/null @@ -1,239 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver.wal; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.util.TreeMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.io.util.LRUDictionary; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.wal.FSHLogProvider; -import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.io.SequenceFile.Metadata; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.compress.CompressionCodec; -import org.apache.hadoop.io.compress.DefaultCodec; - -/** - * Implementation of {@link WALProvider.Writer} that delegates to - * SequenceFile.Writer. Legacy implementation only used for compat tests. - * - * Note that because this class writes to the legacy hadoop-specific SequenceFile - * format, users of it must write {@link HLogKey} keys and not arbitrary - * {@link WALKey}s because the latter are not Writables (nor made to work with - * Hadoop serialization). - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class SequenceFileLogWriter implements FSHLogProvider.Writer { - private static final Log LOG = LogFactory.getLog(SequenceFileLogWriter.class); - // The sequence file we delegate to. - private SequenceFile.Writer writer; - // This is the FSDataOutputStream instance that is the 'out' instance - // in the SequenceFile.Writer 'writer' instance above. - private FSDataOutputStream writer_out; - - private CompressionContext compressionContext; - - // Legacy stuff from pre-PB WAL metadata. - private static final Text WAL_VERSION_KEY = new Text("version"); - private static final Text WAL_COMPRESSION_TYPE_KEY = new Text("compression.type"); - private static final Text DICTIONARY_COMPRESSION_TYPE = new Text("dictionary"); - - /** - * Default constructor. - */ - public SequenceFileLogWriter() { - super(); - } - /** - * Create sequence file Metadata for our WAL file with version and compression - * type (if any). - * @param conf - * @param compress - * @return Metadata instance. - */ - private static Metadata createMetadata(final Configuration conf, - final boolean compress) { - TreeMap metaMap = new TreeMap(); - metaMap.put(WAL_VERSION_KEY, new Text("1")); - if (compress) { - // Currently we only do one compression type. - metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE); - } - return new Metadata(metaMap); - } - - private boolean initializeCompressionContext(Configuration conf, Path path) throws IOException { - boolean doCompress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false); - if (doCompress) { - try { - this.compressionContext = new CompressionContext(LRUDictionary.class, - FSUtils.isRecoveredEdits(path), conf.getBoolean( - CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, true)); - } catch (Exception e) { - throw new IOException("Failed to initiate CompressionContext", e); - } - } - return doCompress; - } - - @Override - public void init(FileSystem fs, Path path, Configuration conf, boolean overwritable) - throws IOException { - boolean compress = initializeCompressionContext(conf, path); - // Create a SF.Writer instance. - try { - // reflection for a version of SequenceFile.createWriter that doesn't - // automatically create the parent directory (see HBASE-2312) - this.writer = (SequenceFile.Writer) SequenceFile.class - .getMethod("createWriter", new Class[] {FileSystem.class, - Configuration.class, Path.class, Class.class, Class.class, - Integer.TYPE, Short.TYPE, Long.TYPE, Boolean.TYPE, - CompressionType.class, CompressionCodec.class, Metadata.class}) - .invoke(null, new Object[] {fs, conf, path, HLogKey.class, WALEdit.class, - Integer.valueOf(FSUtils.getDefaultBufferSize(fs)), - Short.valueOf((short) - conf.getInt("hbase.regionserver.hlog.replication", - FSUtils.getDefaultReplication(fs, path))), - Long.valueOf(conf.getLong("hbase.regionserver.hlog.blocksize", - FSUtils.getDefaultBlockSize(fs, path))), - Boolean.valueOf(false) /*createParent*/, - SequenceFile.CompressionType.NONE, new DefaultCodec(), - createMetadata(conf, compress) - }); - } catch (InvocationTargetException ite) { - // function was properly called, but threw it's own exception - throw new IOException(ite.getCause()); - } catch (Exception e) { - // ignore all other exceptions. related to reflection failure - } - - // if reflection failed, use the old createWriter - if (this.writer == null) { - LOG.debug("new createWriter -- HADOOP-6840 -- not available"); - this.writer = SequenceFile.createWriter(fs, conf, path, - HLogKey.class, WALEdit.class, - FSUtils.getDefaultBufferSize(fs), - (short) conf.getInt("hbase.regionserver.hlog.replication", - FSUtils.getDefaultReplication(fs, path)), - conf.getLong("hbase.regionserver.hlog.blocksize", - FSUtils.getDefaultBlockSize(fs, path)), - SequenceFile.CompressionType.NONE, - new DefaultCodec(), - null, - createMetadata(conf, compress)); - } else { - if (LOG.isTraceEnabled()) LOG.trace("Using new createWriter -- HADOOP-6840"); - } - - this.writer_out = getSequenceFilePrivateFSDataOutputStreamAccessible(); - if (LOG.isTraceEnabled()) LOG.trace("Path=" + path + ", compression=" + compress); - } - - // Get at the private FSDataOutputStream inside in SequenceFile so we can - // call sync on it. Make it accessible. - private FSDataOutputStream getSequenceFilePrivateFSDataOutputStreamAccessible() - throws IOException { - FSDataOutputStream out = null; - final Field fields [] = this.writer.getClass().getDeclaredFields(); - final String fieldName = "out"; - for (int i = 0; i < fields.length; ++i) { - if (fieldName.equals(fields[i].getName())) { - try { - // Make the 'out' field up in SF.Writer accessible. - fields[i].setAccessible(true); - out = (FSDataOutputStream)fields[i].get(this.writer); - break; - } catch (IllegalAccessException ex) { - throw new IOException("Accessing " + fieldName, ex); - } catch (SecurityException e) { - LOG.warn("Does not have access to out field from FSDataOutputStream", - e); - } - } - } - return out; - } - - @Override - public void append(WAL.Entry entry) throws IOException { - entry.setCompressionContext(compressionContext); - try { - this.writer.append(entry.getKey(), entry.getEdit()); - } catch (NullPointerException npe) { - // Concurrent close... - throw new IOException(npe); - } - } - - @Override - public void close() throws IOException { - if (this.writer != null) { - try { - this.writer.close(); - } catch (NullPointerException npe) { - // Can get a NPE coming up from down in DFSClient$DFSOutputStream#close - LOG.warn(npe); - } - this.writer = null; - } - } - - @Override - public void sync() throws IOException { - try { - this.writer.syncFs(); - } catch (NullPointerException npe) { - // Concurrent close... - throw new IOException(npe); - } - } - - @Override - public long getLength() throws IOException { - try { - return this.writer.getLength(); - } catch (NullPointerException npe) { - // Concurrent close... - throw new IOException(npe); - } - } - - /** - * @return The dfsclient out stream up inside SF.Writer made accessible, or - * null if not available. - */ - public FSDataOutputStream getWriterFSDataOutputStream() { - return this.writer_out; - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadOldRootAndMetaEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadOldRootAndMetaEdits.java deleted file mode 100644 index 8bdb33c..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadOldRootAndMetaEdits.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver.wal; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.wal.WAL; -import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.hbase.wal.WALProvider; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Tests to read old ROOT, Meta edits. - */ -@Category({RegionServerTests.class, MediumTests.class}) - -public class TestReadOldRootAndMetaEdits { - - private final static Log LOG = LogFactory.getLog(TestReadOldRootAndMetaEdits.class); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static Configuration conf; - private static FileSystem fs; - private static Path dir; - - @BeforeClass - public static void setupBeforeClass() throws Exception { - conf = TEST_UTIL.getConfiguration(); - conf.setClass("hbase.regionserver.hlog.writer.impl", - SequenceFileLogWriter.class, WALProvider.Writer.class); - fs = TEST_UTIL.getTestFileSystem(); - dir = new Path(TEST_UTIL.createRootDir(), "testReadOldRootAndMetaEdits"); - fs.mkdirs(dir); - - } - @AfterClass - public static void tearDownAfterClass() throws Exception { - } - - /** - * Inserts three waledits in the wal file, and reads them back. The first edit is of a regular - * table, second waledit is for the ROOT table (it will be ignored while reading), - * and last waledit is for the hbase:meta table, which will be linked to the new system:meta table. - * @throws IOException - */ - @Test - public void testReadOldRootAndMetaEdits() throws IOException { - LOG.debug("testReadOldRootAndMetaEdits"); - // kv list to be used for all WALEdits. - byte[] row = Bytes.toBytes("row"); - KeyValue kv = new KeyValue(row, row, row, row); - List kvs = new ArrayList(); - kvs.add(kv); - - WALProvider.Writer writer = null; - WAL.Reader reader = null; - // a regular table - TableName t = TableName.valueOf("t"); - HRegionInfo tRegionInfo = null; - int logCount = 0; - long timestamp = System.currentTimeMillis(); - Path path = new Path(dir, "t"); - try { - tRegionInfo = new HRegionInfo(t, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); - WAL.Entry tEntry = createAEntry(new HLogKey(tRegionInfo.getEncodedNameAsBytes(), t, - ++logCount, timestamp, HConstants.DEFAULT_CLUSTER_ID), kvs); - - // create a old root edit (-ROOT-). - WAL.Entry rootEntry = createAEntry(new HLogKey(Bytes.toBytes(TableName.OLD_ROOT_STR), - TableName.OLD_ROOT_TABLE_NAME, ++logCount, timestamp, - HConstants.DEFAULT_CLUSTER_ID), kvs); - - // create a old meta edit (hbase:meta). - WAL.Entry oldMetaEntry = createAEntry(new HLogKey(Bytes.toBytes(TableName.OLD_META_STR), - TableName.OLD_META_TABLE_NAME, ++logCount, timestamp, - HConstants.DEFAULT_CLUSTER_ID), kvs); - - // write above entries - writer = WALFactory.createWALWriter(fs, path, conf); - writer.append(tEntry); - writer.append(rootEntry); - writer.append(oldMetaEntry); - - // sync/close the writer - writer.sync(); - writer.close(); - - // read the log and see things are okay. - reader = WALFactory.createReader(fs, path, conf); - WAL.Entry entry = reader.next(); - assertNotNull(entry); - assertTrue(entry.getKey().getTablename().equals(t)); - assertEquals(Bytes.toString(entry.getKey().getEncodedRegionName()), - Bytes.toString(tRegionInfo.getEncodedNameAsBytes())); - - // read the ROOT waledit, but that will be ignored, and hbase:meta waledit will be read instead. - entry = reader.next(); - assertEquals(entry.getKey().getTablename(), TableName.META_TABLE_NAME); - // should reach end of log - assertNull(reader.next()); - } finally { - if (writer != null) { - writer.close(); - } - if (reader != null) { - reader.close(); - } - } -} - /** - * Creates a WALEdit for the passed KeyValues and returns a WALProvider.Entry instance composed of - * the WALEdit and passed WALKey. - * @return WAL.Entry instance for the passed WALKey and KeyValues - */ - private WAL.Entry createAEntry(WALKey walKey, List kvs) { - WALEdit edit = new WALEdit(); - for (KeyValue kv : kvs ) - edit.add(kv); - return new WAL.Entry(walKey, edit); - } - -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java index 3d4062f..2dbacaf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java @@ -18,6 +18,12 @@ package org.apache.hadoop.hbase.replication; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.Lists; + import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -30,22 +36,17 @@ import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.hbase.regionserver.wal.WALEdit; -import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; -import com.google.common.collect.Lists; - -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; - @Category({ReplicationTests.class, SmallTests.class}) public class TestReplicationWALEntryFilters { @@ -59,20 +60,21 @@ public class TestReplicationWALEntryFilters { SystemTableWALEntryFilter filter = new SystemTableWALEntryFilter(); // meta - WALKey key1 = new WALKey( HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), - TableName.META_TABLE_NAME, null); + WALKey key1 = new WALKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), + TableName.META_TABLE_NAME, System.currentTimeMillis()); Entry metaEntry = new Entry(key1, null); assertNull(filter.filter(metaEntry)); // ns table - WALKey key2 = new WALKey(new byte[] {}, TableName.NAMESPACE_TABLE_NAME, null); + WALKey key2 = + new WALKey(new byte[0], TableName.NAMESPACE_TABLE_NAME, System.currentTimeMillis()); Entry nsEntry = new Entry(key2, null); assertNull(filter.filter(nsEntry)); // user table - WALKey key3 = new WALKey(new byte[] {}, TableName.valueOf("foo"), null); + WALKey key3 = new WALKey(new byte[0], TableName.valueOf("foo"), System.currentTimeMillis()); Entry userEntry = new Entry(key3, null); assertEquals(userEntry, filter.filter(userEntry)); @@ -298,7 +300,8 @@ public class TestReplicationWALEntryFilters { } private Entry createEntry(TreeMap scopes, byte[]... kvs) { - WALKey key1 = new WALKey(new byte[] {}, TableName.valueOf("foo"), scopes); + WALKey key1 = + new WALKey(new byte[0], TableName.valueOf("foo"), System.currentTimeMillis(), scopes); WALEdit edit1 = new WALEdit(); for (byte[] kv : kvs) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java new file mode 100644 index 0000000..3739d75 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java @@ -0,0 +1,816 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security.access; + +import com.google.protobuf.Message; +import com.google.protobuf.RpcController; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.lang.reflect.Array; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.Action; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.MultiAction; +import org.apache.hadoop.hbase.client.MultiResponse; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.BitComparator; +import org.apache.hadoop.hbase.filter.ByteArrayComparable; +import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; +import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; +import org.apache.hadoop.hbase.filter.ColumnRangeFilter; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.DependentColumnFilter; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.KeyOnlyFilter; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.filter.RandomRowFilter; +import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.SkipFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.filter.WhileMatchFilter; +import org.apache.hadoop.hbase.io.WritableWithSize; +import org.apache.hadoop.hbase.regionserver.RegionOpeningState; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ProtoUtil; +import org.apache.hadoop.hbase.wal.WAL.Entry; +import org.apache.hadoop.io.DataOutputOutputStream; +import org.apache.hadoop.io.MapWritable; +import org.apache.hadoop.io.ObjectWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableFactories; +import org.apache.hadoop.io.WritableUtils; + +/** + *

This is a customized version of the polymorphic hadoop + * {@link ObjectWritable}. It removes UTF8 (HADOOP-414). + * Using {@link Text} intead of UTF-8 saves ~2% CPU between reading and writing + * objects running a short sequentialWrite Performance Evaluation test just in + * ObjectWritable alone; more when we're doing randomRead-ing. Other + * optimizations include our passing codes for classes instead of the + * actual class names themselves. This makes it so this class needs amendment + * if non-Writable classes are introduced -- if passed a Writable for which we + * have no code, we just do the old-school passing of the class name, etc. -- + * but passing codes the savings are large particularly when cell + * data is small (If < a couple of kilobytes, the encoding/decoding of class + * name and reflection to instantiate class was costing in excess of the cell + * handling). + * @deprecated This class is needed migrating TablePermissions written with + * Writables. It is needed to read old permissions written pre-0.96. This + * class is to be removed after HBase 0.96 ships since then all permissions + * will have been migrated and written with protobufs. + */ +@Deprecated +@InterfaceAudience.Private +class HbaseObjectWritableFor96Migration implements Writable, WritableWithSize, Configurable { + private final static Log LOG = LogFactory.getLog(HbaseObjectWritableFor96Migration.class); + + // Here we maintain two static maps of classes to code and vice versa. + // Add new classes+codes as wanted or figure way to auto-generate these + // maps. + static final Map> CODE_TO_CLASS = + new HashMap>(); + static final Map, Integer> CLASS_TO_CODE = + new HashMap, Integer>(); + // Special code that means 'not-encoded'; in this case we do old school + // sending of the class name using reflection, etc. + private static final byte NOT_ENCODED = 0; + //Generic array means that the array type is not one of the pre-defined arrays + //in the CLASS_TO_CODE map, but we have to still encode the array since it's + //elements are serializable by this class. + private static final int GENERIC_ARRAY_CODE; + private static final int NEXT_CLASS_CODE; + static { + //////////////////////////////////////////////////////////////////////////// + // WARNING: Please do not insert, remove or swap any line in this static // + // block. Doing so would change or shift all the codes used to serialize // + // objects, which makes backwards compatibility very hard for clients. // + // New codes should always be added at the end. Code removal is // + // discouraged because code is a short now. // + //////////////////////////////////////////////////////////////////////////// + + int code = NOT_ENCODED + 1; + // Primitive types. + addToMap(Boolean.TYPE, code++); + addToMap(Byte.TYPE, code++); + addToMap(Character.TYPE, code++); + addToMap(Short.TYPE, code++); + addToMap(Integer.TYPE, code++); + addToMap(Long.TYPE, code++); + addToMap(Float.TYPE, code++); + addToMap(Double.TYPE, code++); + addToMap(Void.TYPE, code++); + + // Other java types + addToMap(String.class, code++); + addToMap(byte [].class, code++); + addToMap(byte [][].class, code++); + + // Hadoop types + addToMap(Text.class, code++); + addToMap(Writable.class, code++); + addToMap(Writable [].class, code++); + code++; // Removed + addToMap(NullInstance.class, code++); + + // Hbase types + addToMap(HColumnDescriptor.class, code++); + addToMap(HConstants.Modify.class, code++); + + // We used to have a class named HMsg but its been removed. Rather than + // just axe it, use following random Integer class -- we just chose any + // class from java.lang -- instead just so codes that follow stay + // in same relative place. + addToMap(Integer.class, code++); + addToMap(Integer[].class, code++); + + //HRegion shouldn't be pushed across the wire. + code++; //addToMap(HRegion.class, code++); + code++; //addToMap(HRegion[].class, code++); + + addToMap(HRegionInfo.class, code++); + addToMap(HRegionInfo[].class, code++); + code++; // Removed + code++; // Removed + addToMap(HTableDescriptor.class, code++); + addToMap(MapWritable.class, code++); + + // + // HBASE-880 + // + addToMap(ClusterStatus.class, code++); + addToMap(Delete.class, code++); + addToMap(Get.class, code++); + addToMap(KeyValue.class, code++); + addToMap(KeyValue[].class, code++); + addToMap(Put.class, code++); + addToMap(Put[].class, code++); + addToMap(Result.class, code++); + addToMap(Result[].class, code++); + addToMap(Scan.class, code++); + + addToMap(WhileMatchFilter.class, code++); + addToMap(PrefixFilter.class, code++); + addToMap(PageFilter.class, code++); + addToMap(InclusiveStopFilter.class, code++); + addToMap(ColumnCountGetFilter.class, code++); + addToMap(SingleColumnValueFilter.class, code++); + addToMap(SingleColumnValueExcludeFilter.class, code++); + addToMap(BinaryComparator.class, code++); + addToMap(BitComparator.class, code++); + addToMap(CompareFilter.class, code++); + addToMap(RowFilter.class, code++); + addToMap(ValueFilter.class, code++); + addToMap(QualifierFilter.class, code++); + addToMap(SkipFilter.class, code++); + addToMap(ByteArrayComparable.class, code++); + addToMap(FirstKeyOnlyFilter.class, code++); + addToMap(DependentColumnFilter.class, code++); + + addToMap(Delete [].class, code++); + + addToMap(Entry.class, code++); + addToMap(Entry[].class, code++); + + // For HLogKey + code++; + addToMap(List.class, code++); + + addToMap(NavigableSet.class, code++); + addToMap(ColumnPrefixFilter.class, code++); + + // Multi + addToMap(Row.class, code++); + addToMap(Action.class, code++); + addToMap(MultiAction.class, code++); + addToMap(MultiResponse.class, code++); + + // coprocessor execution + // Exec no longer exists --> addToMap(Exec.class, code++); + code++; + addToMap(Increment.class, code++); + + addToMap(KeyOnlyFilter.class, code++); + + // serializable + addToMap(Serializable.class, code++); + + addToMap(RandomRowFilter.class, code++); + + addToMap(CompareOp.class, code++); + + addToMap(ColumnRangeFilter.class, code++); + + // HServerLoad no longer exists; increase code so other classes stay the same. + code++; + //addToMap(HServerLoad.class, code++); + + addToMap(RegionOpeningState.class, code++); + + addToMap(HTableDescriptor[].class, code++); + + addToMap(Append.class, code++); + + addToMap(RowMutations.class, code++); + + addToMap(Message.class, code++); + + //java.lang.reflect.Array is a placeholder for arrays not defined above + GENERIC_ARRAY_CODE = code++; + addToMap(Array.class, GENERIC_ARRAY_CODE); + + addToMap(RpcController.class, code++); + + // make sure that this is the last statement in this static block + NEXT_CLASS_CODE = code; + } + + private Class declaredClass; + private Object instance; + private Configuration conf; + + /** default constructor for writable */ + HbaseObjectWritableFor96Migration() { + super(); + } + + /** + * @param instance + */ + HbaseObjectWritableFor96Migration(Object instance) { + set(instance); + } + + /** + * @param declaredClass + * @param instance + */ + HbaseObjectWritableFor96Migration(Class declaredClass, Object instance) { + this.declaredClass = declaredClass; + this.instance = instance; + } + + /** @return the instance, or null if none. */ + Object get() { return instance; } + + /** @return the class this is meant to be. */ + Class getDeclaredClass() { return declaredClass; } + + /** + * Reset the instance. + * @param instance + */ + void set(Object instance) { + this.declaredClass = instance.getClass(); + this.instance = instance; + } + + /** + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "OW[class=" + declaredClass + ",value=" + instance + "]"; + } + + + public void readFields(DataInput in) throws IOException { + readObject(in, this, this.conf); + } + + public void write(DataOutput out) throws IOException { + writeObject(out, instance, declaredClass, conf); + } + + public long getWritableSize() { + return getWritableSize(instance, declaredClass, conf); + } + + private static class NullInstance extends Configured implements Writable { + Class declaredClass; + /** default constructor for writable */ + @SuppressWarnings("unused") + public NullInstance() { super(null); } + + /** + * @param declaredClass + * @param conf + */ + public NullInstance(Class declaredClass, Configuration conf) { + super(conf); + this.declaredClass = declaredClass; + } + + public void readFields(DataInput in) throws IOException { + this.declaredClass = CODE_TO_CLASS.get(WritableUtils.readVInt(in)); + } + + public void write(DataOutput out) throws IOException { + writeClassCode(out, this.declaredClass); + } + } + + static Integer getClassCode(final Class c) + throws IOException { + Integer code = CLASS_TO_CODE.get(c); + if (code == null ) { + if (List.class.isAssignableFrom(c)) { + code = CLASS_TO_CODE.get(List.class); + } else if (Writable.class.isAssignableFrom(c)) { + code = CLASS_TO_CODE.get(Writable.class); + } else if (c.isArray()) { + code = CLASS_TO_CODE.get(Array.class); + } else if (Message.class.isAssignableFrom(c)) { + code = CLASS_TO_CODE.get(Message.class); + } else if (Serializable.class.isAssignableFrom(c)){ + code = CLASS_TO_CODE.get(Serializable.class); + } else if (Scan.class.isAssignableFrom(c)) { + code = CLASS_TO_CODE.get(Scan.class); + } + } + return code; + } + + /** + * @return the next object code in the list. Used in testing to verify that additional fields are not added + */ + static int getNextClassCode(){ + return NEXT_CLASS_CODE; + } + + /** + * Write out the code for passed Class. + * @param out + * @param c + * @throws IOException + */ + static void writeClassCode(final DataOutput out, final Class c) + throws IOException { + Integer code = getClassCode(c); + + if (code == null) { + LOG.error("Unsupported type " + c); + StackTraceElement[] els = new Exception().getStackTrace(); + for(StackTraceElement elem : els) { + LOG.error(elem.getMethodName()); + } + throw new UnsupportedOperationException("No code for unexpected " + c); + } + WritableUtils.writeVInt(out, code); + } + + static long getWritableSize(Object instance, Class declaredClass, + Configuration conf) { + return 0L; // no hint is the default. + } + /** + * Write a {@link Writable}, {@link String}, primitive type, or an array of + * the preceding. + * @param out + * @param instance + * @param declaredClass + * @param conf + * @throws IOException + */ + @SuppressWarnings("unchecked") + static void writeObject(DataOutput out, Object instance, + Class declaredClass, + Configuration conf) + throws IOException { + + Object instanceObj = instance; + Class declClass = declaredClass; + + if (instanceObj == null) { // null + instanceObj = new NullInstance(declClass, conf); + declClass = Writable.class; + } + writeClassCode(out, declClass); + if (declClass.isArray()) { // array + // If bytearray, just dump it out -- avoid the recursion and + // byte-at-a-time we were previously doing. + if (declClass.equals(byte [].class)) { + Bytes.writeByteArray(out, (byte [])instanceObj); + } else { + //if it is a Generic array, write the element's type + if (getClassCode(declaredClass) == GENERIC_ARRAY_CODE) { + Class componentType = declaredClass.getComponentType(); + writeClass(out, componentType); + } + + int length = Array.getLength(instanceObj); + out.writeInt(length); + for (int i = 0; i < length; i++) { + Object item = Array.get(instanceObj, i); + writeObject(out, item, + item.getClass(), conf); + } + } + } else if (List.class.isAssignableFrom(declClass)) { + List list = (List)instanceObj; + int length = list.size(); + out.writeInt(length); + for (int i = 0; i < length; i++) { + Object elem = list.get(i); + writeObject(out, elem, + elem == null ? Writable.class : elem.getClass(), conf); + } + } else if (declClass == String.class) { // String + Text.writeString(out, (String)instanceObj); + } else if (declClass.isPrimitive()) { // primitive type + if (declClass == Boolean.TYPE) { // boolean + out.writeBoolean(((Boolean)instanceObj).booleanValue()); + } else if (declClass == Character.TYPE) { // char + out.writeChar(((Character)instanceObj).charValue()); + } else if (declClass == Byte.TYPE) { // byte + out.writeByte(((Byte)instanceObj).byteValue()); + } else if (declClass == Short.TYPE) { // short + out.writeShort(((Short)instanceObj).shortValue()); + } else if (declClass == Integer.TYPE) { // int + out.writeInt(((Integer)instanceObj).intValue()); + } else if (declClass == Long.TYPE) { // long + out.writeLong(((Long)instanceObj).longValue()); + } else if (declClass == Float.TYPE) { // float + out.writeFloat(((Float)instanceObj).floatValue()); + } else if (declClass == Double.TYPE) { // double + out.writeDouble(((Double)instanceObj).doubleValue()); + } else if (declClass == Void.TYPE) { // void + } else { + throw new IllegalArgumentException("Not a primitive: "+declClass); + } + } else if (declClass.isEnum()) { // enum + Text.writeString(out, ((Enum)instanceObj).name()); + } else if (Message.class.isAssignableFrom(declaredClass)) { + Text.writeString(out, instanceObj.getClass().getName()); + ((Message)instance).writeDelimitedTo( + DataOutputOutputStream.constructOutputStream(out)); + } else if (Writable.class.isAssignableFrom(declClass)) { // Writable + Class c = instanceObj.getClass(); + Integer code = CLASS_TO_CODE.get(c); + if (code == null) { + out.writeByte(NOT_ENCODED); + Text.writeString(out, c.getName()); + } else { + writeClassCode(out, c); + } + ((Writable)instanceObj).write(out); + } else if (Serializable.class.isAssignableFrom(declClass)) { + Class c = instanceObj.getClass(); + Integer code = CLASS_TO_CODE.get(c); + if (code == null) { + out.writeByte(NOT_ENCODED); + Text.writeString(out, c.getName()); + } else { + writeClassCode(out, c); + } + ByteArrayOutputStream bos = null; + ObjectOutputStream oos = null; + try{ + bos = new ByteArrayOutputStream(); + oos = new ObjectOutputStream(bos); + oos.writeObject(instanceObj); + byte[] value = bos.toByteArray(); + out.writeInt(value.length); + out.write(value); + } finally { + if(bos!=null) bos.close(); + if(oos!=null) oos.close(); + } + } else if (Scan.class.isAssignableFrom(declClass)) { + Scan scan = (Scan)instanceObj; + byte [] scanBytes = ProtobufUtil.toScan(scan).toByteArray(); + out.writeInt(scanBytes.length); + out.write(scanBytes); + } else { + throw new IOException("Can't write: "+instanceObj+" as "+declClass); + } + } + + /** Writes the encoded class code as defined in CLASS_TO_CODE, or + * the whole class name if not defined in the mapping. + */ + static void writeClass(DataOutput out, Class c) throws IOException { + Integer code = CLASS_TO_CODE.get(c); + if (code == null) { + WritableUtils.writeVInt(out, NOT_ENCODED); + Text.writeString(out, c.getName()); + } else { + WritableUtils.writeVInt(out, code); + } + } + + /** Reads and returns the class as written by {@link #writeClass(DataOutput, Class)} */ + static Class readClass(Configuration conf, DataInput in) throws IOException { + Class instanceClass = null; + int b = (byte)WritableUtils.readVInt(in); + if (b == NOT_ENCODED) { + String className = Text.readString(in); + try { + instanceClass = getClassByName(conf, className); + } catch (ClassNotFoundException e) { + LOG.error("Can't find class " + className, e); + throw new IOException("Can't find class " + className, e); + } + } else { + instanceClass = CODE_TO_CLASS.get(b); + } + return instanceClass; + } + + /** + * Read a {@link Writable}, {@link String}, primitive type, or an array of + * the preceding. + * @param in + * @param conf + * @return the object + * @throws IOException + */ + static Object readObject(DataInput in, Configuration conf) + throws IOException { + return readObject(in, null, conf); + } + + /** + * Read a {@link Writable}, {@link String}, primitive type, or an array of + * the preceding. + * @param in + * @param objectWritable + * @param conf + * @return the object + * @throws IOException + */ + @SuppressWarnings("unchecked") + static Object readObject(DataInput in, + HbaseObjectWritableFor96Migration objectWritable, Configuration conf) + throws IOException { + Class declaredClass = CODE_TO_CLASS.get(WritableUtils.readVInt(in)); + Object instance; + if (declaredClass.isPrimitive()) { // primitive types + if (declaredClass == Boolean.TYPE) { // boolean + instance = Boolean.valueOf(in.readBoolean()); + } else if (declaredClass == Character.TYPE) { // char + instance = Character.valueOf(in.readChar()); + } else if (declaredClass == Byte.TYPE) { // byte + instance = Byte.valueOf(in.readByte()); + } else if (declaredClass == Short.TYPE) { // short + instance = Short.valueOf(in.readShort()); + } else if (declaredClass == Integer.TYPE) { // int + instance = Integer.valueOf(in.readInt()); + } else if (declaredClass == Long.TYPE) { // long + instance = Long.valueOf(in.readLong()); + } else if (declaredClass == Float.TYPE) { // float + instance = Float.valueOf(in.readFloat()); + } else if (declaredClass == Double.TYPE) { // double + instance = Double.valueOf(in.readDouble()); + } else if (declaredClass == Void.TYPE) { // void + instance = null; + } else { + throw new IllegalArgumentException("Not a primitive: "+declaredClass); + } + } else if (declaredClass.isArray()) { // array + if (declaredClass.equals(byte [].class)) { + instance = Bytes.readByteArray(in); + } else { + int length = in.readInt(); + instance = Array.newInstance(declaredClass.getComponentType(), length); + for (int i = 0; i < length; i++) { + Array.set(instance, i, readObject(in, conf)); + } + } + } else if (declaredClass.equals(Array.class)) { //an array not declared in CLASS_TO_CODE + Class componentType = readClass(conf, in); + int length = in.readInt(); + instance = Array.newInstance(componentType, length); + for (int i = 0; i < length; i++) { + Array.set(instance, i, readObject(in, conf)); + } + } else if (List.class.isAssignableFrom(declaredClass)) { // List + int length = in.readInt(); + instance = new ArrayList(length); + for (int i = 0; i < length; i++) { + ((ArrayList)instance).add(readObject(in, conf)); + } + } else if (declaredClass == String.class) { // String + instance = Text.readString(in); + } else if (declaredClass.isEnum()) { // enum + instance = Enum.valueOf((Class) declaredClass, + Text.readString(in)); + } else if (declaredClass == Message.class) { + String className = Text.readString(in); + try { + declaredClass = getClassByName(conf, className); + instance = tryInstantiateProtobuf(declaredClass, in); + } catch (ClassNotFoundException e) { + LOG.error("Can't find class " + className, e); + throw new IOException("Can't find class " + className, e); + } + } else if (Scan.class.isAssignableFrom(declaredClass)) { + int length = in.readInt(); + byte [] scanBytes = new byte[length]; + in.readFully(scanBytes); + ClientProtos.Scan.Builder scanProto = ClientProtos.Scan.newBuilder(); + ProtobufUtil.mergeFrom(scanProto, scanBytes); + instance = ProtobufUtil.toScan(scanProto.build()); + } else { // Writable or Serializable + Class instanceClass = null; + int b = (byte)WritableUtils.readVInt(in); + if (b == NOT_ENCODED) { + String className = Text.readString(in); + if ("org.apache.hadoop.hbase.regionserver.wal.HLog$Entry".equals(className)) { + className = Entry.class.getName(); + } + try { + instanceClass = getClassByName(conf, className); + } catch (ClassNotFoundException e) { + LOG.error("Can't find class " + className, e); + throw new IOException("Can't find class " + className, e); + } + } else { + instanceClass = CODE_TO_CLASS.get(b); + } + if(Writable.class.isAssignableFrom(instanceClass)){ + Writable writable = WritableFactories.newInstance(instanceClass, conf); + try { + writable.readFields(in); + } catch (Exception e) { + LOG.error("Error in readFields", e); + throw new IOException("Error in readFields" , e); + } + instance = writable; + if (instanceClass == NullInstance.class) { // null + declaredClass = ((NullInstance)instance).declaredClass; + instance = null; + } + } else { + int length = in.readInt(); + byte[] objectBytes = new byte[length]; + in.readFully(objectBytes); + ByteArrayInputStream bis = null; + ObjectInputStream ois = null; + try { + bis = new ByteArrayInputStream(objectBytes); + ois = new ObjectInputStream(bis); + instance = ois.readObject(); + } catch (ClassNotFoundException e) { + LOG.error("Class not found when attempting to deserialize object", e); + throw new IOException("Class not found when attempting to " + + "deserialize object", e); + } finally { + if(bis!=null) bis.close(); + if(ois!=null) ois.close(); + } + } + } + if (objectWritable != null) { // store values + objectWritable.declaredClass = declaredClass; + objectWritable.instance = instance; + } + return instance; + } + + /** + * Try to instantiate a protocol buffer of the given message class + * from the given input stream. + * + * @param protoClass the class of the generated protocol buffer + * @param dataIn the input stream to read from + * @return the instantiated Message instance + * @throws IOException if an IO problem occurs + */ + static Message tryInstantiateProtobuf( + Class protoClass, + DataInput dataIn) throws IOException { + + try { + if (dataIn instanceof InputStream) { + // We can use the built-in parseDelimitedFrom and not have to re-copy + // the data + Method parseMethod = getStaticProtobufMethod(protoClass, + "parseDelimitedFrom", InputStream.class); + return (Message)parseMethod.invoke(null, (InputStream)dataIn); + } else { + // Have to read it into a buffer first, since protobuf doesn't deal + // with the DataInput interface directly. + + // Read the size delimiter that writeDelimitedTo writes + int size = ProtoUtil.readRawVarint32(dataIn); + if (size < 0) { + throw new IOException("Invalid size: " + size); + } + + byte[] data = new byte[size]; + dataIn.readFully(data); + Method parseMethod = getStaticProtobufMethod(protoClass, + "parseFrom", byte[].class); + return (Message)parseMethod.invoke(null, data); + } + } catch (InvocationTargetException e) { + + if (e.getCause() instanceof IOException) { + throw (IOException)e.getCause(); + } else { + throw new IOException(e.getCause()); + } + } catch (IllegalAccessException iae) { + throw new AssertionError("Could not access parse method in " + + protoClass); + } + } + + static Method getStaticProtobufMethod(Class declaredClass, String method, + Class ... args) { + + try { + return declaredClass.getMethod(method, args); + } catch (Exception e) { + // This is a bug in Hadoop - protobufs should all have this static method + throw new AssertionError("Protocol buffer class " + declaredClass + + " does not have an accessible parseFrom(InputStream) method!"); + } + } + + @SuppressWarnings("unchecked") + private static Class getClassByName(Configuration conf, String className) + throws ClassNotFoundException { + if(conf != null) { + return conf.getClassByName(className); + } + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + if(cl == null) { + cl = HbaseObjectWritableFor96Migration.class.getClassLoader(); + } + return Class.forName(className, true, cl); + } + + private static void addToMap(final Class clazz, final int code) { + CLASS_TO_CODE.put(clazz, code); + CODE_TO_CLASS.put(code, clazz); + } + + public void setConf(Configuration conf) { + this.conf = conf; + } + + public Configuration getConf() { + return this.conf; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java index 65aa73f..68fffb1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestTablePermissions.java @@ -162,8 +162,7 @@ public class TestTablePermissions { * @throws IOException */ public static void writePermissions(DataOutput out, - ListMultimap perms, Configuration conf) - throws IOException { + ListMultimap perms, Configuration conf) throws IOException { Set keys = perms.keySet(); out.writeInt(keys.size()); for (String key : keys) { @@ -172,7 +171,6 @@ public class TestTablePermissions { } } - @Test public void testBasicWrite() throws Exception { Configuration conf = UTIL.getConfiguration(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java deleted file mode 100644 index 15b419c..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.wal; - - -import java.util.NavigableMap; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; - -@Category({RegionServerTests.class, LargeTests.class}) -public class TestDefaultWALProviderWithHLogKey extends TestFSHLogProvider { - @Override - WALKey getWalKey(final byte[] info, final TableName tableName, final long timestamp, - final NavigableMap scopes) { - return new HLogKey(info, tableName, timestamp, mvcc, scopes); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestFSHLogProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestFSHLogProvider.java index 368aa89..7d78e6a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestFSHLogProvider.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestFSHLogProvider.java @@ -29,7 +29,6 @@ import java.util.NavigableMap; import java.util.Random; import java.util.Set; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -207,7 +206,6 @@ public class TestFSHLogProvider { final Configuration localConf = new Configuration(conf); localConf.set(WALFactory.WAL_PROVIDER, FSHLogProvider.class.getName()); final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName()); - final AtomicLong sequenceId = new AtomicLong(1); try { HRegionInfo hri = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java index 74445cf..b95176b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java @@ -18,11 +18,9 @@ */ package org.apache.hadoop.hbase.wal; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -54,9 +52,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader; -import org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -347,9 +342,9 @@ public class TestWALFactory { if (previousRegion != null) { assertEquals(previousRegion, region); } - LOG.info("oldseqno=" + seqno + ", newseqno=" + key.getLogSeqNum()); - assertTrue(seqno < key.getLogSeqNum()); - seqno = key.getLogSeqNum(); + LOG.info("oldseqno=" + seqno + ", newseqno=" + key.getSequenceId()); + assertTrue(seqno < key.getSequenceId()); + seqno = key.getSequenceId(); previousRegion = region; count++; } @@ -675,74 +670,6 @@ public class TestWALFactory { assertNotNull(c); } - /** - * @throws IOException - */ - @Test - public void testReadLegacyLog() throws IOException { - final int columnCount = 5; - final int recordCount = 5; - final TableName tableName = - TableName.valueOf("tablename"); - final byte[] row = Bytes.toBytes("row"); - long timestamp = System.currentTimeMillis(); - Path path = new Path(dir, "tempwal"); - SequenceFileLogWriter sflw = null; - WAL.Reader reader = null; - try { - HRegionInfo hri = new HRegionInfo(tableName, - HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); - HTableDescriptor htd = new HTableDescriptor(tableName); - fs.mkdirs(dir); - // Write log in pre-PB format. - sflw = new SequenceFileLogWriter(); - sflw.init(fs, path, conf, false); - for (int i = 0; i < recordCount; ++i) { - WALKey key = new HLogKey( - hri.getEncodedNameAsBytes(), tableName, i, timestamp, HConstants.DEFAULT_CLUSTER_ID); - WALEdit edit = new WALEdit(); - for (int j = 0; j < columnCount; ++j) { - if (i == 0) { - htd.addFamily(new HColumnDescriptor("column" + j)); - } - String value = i + "" + j; - edit.add(new KeyValue(row, row, row, timestamp, Bytes.toBytes(value))); - } - sflw.append(new WAL.Entry(key, edit)); - } - sflw.sync(); - sflw.close(); - - // Now read the log using standard means. - reader = wals.createReader(fs, path); - assertTrue(reader instanceof SequenceFileLogReader); - for (int i = 0; i < recordCount; ++i) { - WAL.Entry entry = reader.next(); - assertNotNull(entry); - assertEquals(columnCount, entry.getEdit().size()); - assertArrayEquals(hri.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName()); - assertEquals(tableName, entry.getKey().getTablename()); - int idx = 0; - for (Cell val : entry.getEdit().getCells()) { - assertTrue(Bytes.equals(row, 0, row.length, val.getRowArray(), val.getRowOffset(), - val.getRowLength())); - String value = i + "" + idx; - assertArrayEquals(Bytes.toBytes(value), CellUtil.cloneValue(val)); - idx++; - } - } - WAL.Entry entry = reader.next(); - assertNull(entry); - } finally { - if (sflw != null) { - sflw.close(); - } - if (reader != null) { - reader.close(); - } - } - } - static class DumbWALActionsListener extends WALActionsListener.Base { int increments = 0; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java index 1da8892..f45da75 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java @@ -18,7 +18,11 @@ */ package org.apache.hadoop.hbase.wal; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.NavigableSet; @@ -27,21 +31,22 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; -import org.apache.hadoop.hbase.wal.WALSplitter.EntryBuffers; -import org.apache.hadoop.hbase.wal.WALSplitter.PipelineController; -import org.apache.hadoop.hbase.wal.WALSplitter.RegionEntryBuffer; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValueTestUtil; +import org.apache.hadoop.hbase.TableName; +// imports for things that haven't moved from regionserver.wal yet. +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.wal.WALSplitter.EntryBuffers; +import org.apache.hadoop.hbase.wal.WALSplitter.PipelineController; +import org.apache.hadoop.hbase.wal.WALSplitter.RegionEntryBuffer; import org.junit.Test; import org.junit.experimental.categories.Category; -// imports for things that haven't moved from regionserver.wal yet. -import org.apache.hadoop.hbase.regionserver.wal.WALEdit; - /** * Simple testing of a few WAL methods. */ @@ -119,10 +124,6 @@ public class TestWALMethods { @Test public void testEntrySink() throws Exception { - Configuration conf = new Configuration(); - RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? - RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING); - EntryBuffers sink = new EntryBuffers(new PipelineController(), 1*1024*1024); for (int i = 0; i < 1000; i++) { WAL.Entry entry = createTestLogEntry(i); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index de8d9e1..055e4f5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -43,6 +43,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -62,16 +63,16 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.wal.FaultySequenceFileLogReader; +import org.apache.hadoop.hbase.regionserver.wal.FaultyProtobufLogReader; import org.apache.hadoop.hbase.regionserver.wal.InstrumentedLogWriter; import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -229,7 +230,7 @@ public class TestWALSplit { while (startCount == counter.get()) Threads.sleep(1); // Give it a second to write a few appends. Threads.sleep(1000); - final Configuration conf2 = HBaseConfiguration.create(this.conf); + final Configuration conf2 = HBaseConfiguration.create(conf); final User robber = User.createUserForTesting(conf2, ROBBER, GROUP); int count = robber.runAs(new PrivilegedExceptionAction() { @Override @@ -571,11 +572,13 @@ public class TestWALSplit { REGIONS.size() * (goodEntries + firstHalfEntries) <= allRegionsCount); } - @Test (timeout=300000) + @Test(timeout = 300000) public void testCorruptedFileGetsArchivedIfSkipErrors() throws IOException { conf.setBoolean(HBASE_SKIP_ERRORS, true); - for (FaultySequenceFileLogReader.FailureType failureType : - FaultySequenceFileLogReader.FailureType.values()) { + List failureTypes = Arrays + .asList(FaultyProtobufLogReader.FailureType.values()).stream() + .filter(x -> x != FaultyProtobufLogReader.FailureType.NONE).collect(Collectors.toList()); + for (FaultyProtobufLogReader.FailureType failureType : failureTypes) { final Set walDirContents = splitCorruptWALs(failureType); final Set archivedLogs = new HashSet(); final StringBuilder archived = new StringBuilder("Archived logs in CORRUPTDIR:"); @@ -585,7 +588,7 @@ public class TestWALSplit { } LOG.debug(archived.toString()); assertEquals(failureType.name() + ": expected to find all of our wals corrupt.", - walDirContents, archivedLogs); + walDirContents, archivedLogs); } } @@ -593,16 +596,16 @@ public class TestWALSplit { * @return set of wal names present prior to split attempt. * @throws IOException if the split process fails */ - private Set splitCorruptWALs(final FaultySequenceFileLogReader.FailureType failureType) + private Set splitCorruptWALs(final FaultyProtobufLogReader.FailureType failureType) throws IOException { Class backupClass = conf.getClass("hbase.regionserver.hlog.reader.impl", Reader.class); InstrumentedLogWriter.activateFailure = false; try { - conf.setClass("hbase.regionserver.hlog.reader.impl", - FaultySequenceFileLogReader.class, Reader.class); - conf.set("faultysequencefilelogreader.failuretype", failureType.name()); + conf.setClass("hbase.regionserver.hlog.reader.impl", FaultyProtobufLogReader.class, + Reader.class); + conf.set("faultyprotobuflogreader.failuretype", failureType.name()); // Clean up from previous tests or previous loop try { wals.shutdown(); @@ -639,7 +642,7 @@ public class TestWALSplit { public void testTrailingGarbageCorruptionLogFileSkipErrorsFalseThrows() throws IOException { conf.setBoolean(HBASE_SKIP_ERRORS, false); - splitCorruptWALs(FaultySequenceFileLogReader.FailureType.BEGINNING); + splitCorruptWALs(FaultyProtobufLogReader.FailureType.BEGINNING); } @Test (timeout=300000) @@ -647,7 +650,7 @@ public class TestWALSplit { throws IOException { conf.setBoolean(HBASE_SKIP_ERRORS, false); try { - splitCorruptWALs(FaultySequenceFileLogReader.FailureType.BEGINNING); + splitCorruptWALs(FaultyProtobufLogReader.FailureType.BEGINNING); } catch (IOException e) { LOG.debug("split with 'skip errors' set to 'false' correctly threw"); } @@ -1396,11 +1399,12 @@ public class TestWALSplit { HConstants.DEFAULT_CLUSTER_ID), edit); } - private void injectEmptyFile(String suffix, boolean closeFile) - throws IOException { - Writer writer = wals.createWALWriter(fs, new Path(WALDIR, WAL_FILE_PREFIX + suffix), - conf); - if (closeFile) writer.close(); + private void injectEmptyFile(String suffix, boolean closeFile) throws IOException { + Writer writer = + WALFactory.createWALWriter(fs, new Path(WALDIR, WAL_FILE_PREFIX + suffix), conf); + if (closeFile) { + writer.close(); + } } private boolean logsAreEqual(Path p1, Path p2) throws IOException { -- 2.7.4