commit 57e479d60c71a0e87e8b8b82f756bb8b4cf1e2c3 Author: Enis Soztutar Date: Thu Jul 28 20:10:24 2016 -0700 HBASE-16476 Remove HLogKey diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java index 748268e..aceef7e 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java @@ -57,7 +57,6 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WALKey; @@ -470,12 +469,6 @@ public class BaseRegionObserver implements RegionObserver { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { } - @Override - public void preWALRestore(ObserverContext env, HRegionInfo info, - HLogKey logKey, WALEdit logEdit) throws IOException { - preWALRestore(env, info, (WALKey)logKey, logEdit); - } - /** * Implementers should override this version of the method and leave the deprecated one as-is. */ @@ -485,12 +478,6 @@ public class BaseRegionObserver implements RegionObserver { } @Override - public void postWALRestore(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - postWALRestore(env, info, (WALKey)logKey, logEdit); - } - - @Override public void preBulkLoadHFile(final ObserverContext ctx, List> familyPaths) throws IOException { } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseWALObserver.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseWALObserver.java index 1d0076a..840533d 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseWALObserver.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseWALObserver.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -53,12 +52,6 @@ public class BaseWALObserver implements WALObserver { return false; } - @Override - public boolean preWALWrite(ObserverContext ctx, HRegionInfo info, - HLogKey logKey, WALEdit logEdit) throws IOException { - return preWALWrite(ctx, info, (WALKey)logKey, logEdit); - } - /** * Implementers should override this method and leave the deprecated version as-is. */ @@ -67,12 +60,6 @@ public class BaseWALObserver implements WALObserver { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { } @Override - public void postWALWrite(ObserverContext ctx, HRegionInfo info, - HLogKey logKey, WALEdit logEdit) throws IOException { - postWALWrite(ctx, info, (WALKey)logKey, logEdit); - } - - @Override public void preWALRoll(ObserverContext ctx, Path oldPath, Path newPath) throws IOException { } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java index 4c94644..2a76869 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -57,7 +57,6 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileReader; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Pair; @@ -1248,30 +1247,6 @@ public interface RegionObserver extends Coprocessor { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException; /** - * Called before a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} - * replayed for this region. - * - * This method is left in place to maintain binary compatibility with older - * {@link RegionObserver}s. If an implementation directly overrides - * {@link #preWALRestore(ObserverContext, HRegionInfo, WALKey, WALEdit)} then this version - * won't be called at all, barring problems with the Security Manager. To work correctly - * in the presence of a strict Security Manager, or in the case of an implementation that - * relies on a parent class to implement preWALRestore, you should implement this method - * as a call to the non-deprecated version. - * - * Users of this method will see all edits that can be treated as HLogKey. If there are - * edits that can't be treated as HLogKey they won't be offered to coprocessors that rely - * on this method. If a coprocessor gets skipped because of this mechanism, a log message - * at ERROR will be generated per coprocessor on the logger for {@link CoprocessorHost} once per - * classloader. - * - * @deprecated use {@link #preWALRestore(ObserverContext, HRegionInfo, WALKey, WALEdit)} - */ - @Deprecated - void preWALRestore(final ObserverContext ctx, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; - - /** * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} * replayed for this region. */ @@ -1279,30 +1254,6 @@ public interface RegionObserver extends Coprocessor { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException; /** - * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} - * replayed for this region. - * - * This method is left in place to maintain binary compatibility with older - * {@link RegionObserver}s. If an implementation directly overrides - * {@link #postWALRestore(ObserverContext, HRegionInfo, WALKey, WALEdit)} then this version - * won't be called at all, barring problems with the Security Manager. To work correctly - * in the presence of a strict Security Manager, or in the case of an implementation that - * relies on a parent class to implement preWALRestore, you should implement this method - * as a call to the non-deprecated version. - * - * Users of this method will see all edits that can be treated as HLogKey. If there are - * edits that can't be treated as HLogKey they won't be offered to coprocessors that rely - * on this method. If a coprocessor gets skipped because of this mechanism, a log message - * at ERROR will be generated per coprocessor on the logger for {@link CoprocessorHost} once per - * classloader. - * - * @deprecated use {@link #postWALRestore(ObserverContext, HRegionInfo, WALKey, WALEdit)} - */ - @Deprecated - void postWALRestore(final ObserverContext ctx, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; - - /** * Called before bulkLoadHFile. Users can create a StoreFile instance to * access the contents of a HFile. * diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java index 7fd03ce..b84bf65 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -59,31 +58,6 @@ public interface WALObserver extends Coprocessor { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException; /** - * Called before a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} - * is writen to WAL. - * - * This method is left in place to maintain binary compatibility with older - * {@link WALObserver}s. If an implementation directly overrides - * {@link #preWALWrite(ObserverContext, HRegionInfo, WALKey, WALEdit)} then this version - * won't be called at all, barring problems with the Security Manager. To work correctly - * in the presence of a strict Security Manager, or in the case of an implementation that - * relies on a parent class to implement preWALWrite, you should implement this method - * as a call to the non-deprecated version. - * - * Users of this method will see all edits that can be treated as HLogKey. If there are - * edits that can't be treated as HLogKey they won't be offered to coprocessors that rely - * on this method. If a coprocessor gets skipped because of this mechanism, a log message - * at ERROR will be generated per coprocessor on the logger for {@link CoprocessorHost} once per - * classloader. - * - * @return true if default behavior should be bypassed, false otherwise - * @deprecated use {@link #preWALWrite(ObserverContext, HRegionInfo, WALKey, WALEdit)} - */ - @Deprecated - boolean preWALWrite(ObserverContext ctx, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; - - /** * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} * is writen to WAL. */ @@ -91,30 +65,6 @@ public interface WALObserver extends Coprocessor { HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException; /** - * Called after a {@link org.apache.hadoop.hbase.regionserver.wal.WALEdit} - * is writen to WAL. - * - * This method is left in place to maintain binary compatibility with older - * {@link WALObserver}s. If an implementation directly overrides - * {@link #postWALWrite(ObserverContext, HRegionInfo, WALKey, WALEdit)} then this version - * won't be called at all, barring problems with the Security Manager. To work correctly - * in the presence of a strict Security Manager, or in the case of an implementation that - * relies on a parent class to implement preWALWrite, you should implement this method - * as a call to the non-deprecated version. - * - * Users of this method will see all edits that can be treated as HLogKey. If there are - * edits that can't be treated as HLogKey they won't be offered to coprocessors that rely - * on this method. If a coprocessor gets skipped because of this mechanism, a log message - * at ERROR will be generated per coprocessor on the logger for {@link CoprocessorHost} once per - * classloader. - * - * @deprecated use {@link #postWALWrite(ObserverContext, HRegionInfo, WALKey, WALEdit)} - */ - @Deprecated - void postWALWrite(ObserverContext ctx, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException; - - /** * Called before rolling the current WAL * @param oldPath the path of the current wal that we are replacing * @param newPath the path of the wal we are going to create diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java deleted file mode 100644 index e727e15..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HLogInputFormat.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.mapreduce; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.regionserver.wal.WALEdit; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -/** - * Simple {@link InputFormat} for {@link org.apache.hadoop.hbase.wal.WAL} - * files. - * @deprecated use {@link WALInputFormat}. Remove in hadoop 3.0 - */ -@Deprecated -@InterfaceAudience.Public -public class HLogInputFormat extends InputFormat { - private static final Log LOG = LogFactory.getLog(HLogInputFormat.class); - public static final String START_TIME_KEY = "hlog.start.time"; - public static final String END_TIME_KEY = "hlog.end.time"; - - // Delegate to WALInputFormat for implementation. - private final WALInputFormat delegate = new WALInputFormat(); - - /** - * {@link RecordReader} that pulls out the legacy HLogKey format directly. - */ - static class HLogKeyRecordReader extends WALInputFormat.WALRecordReader { - @Override - public HLogKey getCurrentKey() throws IOException, InterruptedException { - if (!(currentEntry.getKey() instanceof HLogKey)) { - final IllegalStateException exception = new IllegalStateException( - "HLogInputFormat only works when given entries that have HLogKey for keys. This" + - " one had '" + currentEntry.getKey().getClass() + "'"); - LOG.error("The deprecated HLogInputFormat has to work with the deprecated HLogKey class, " + - " but HBase internals read the wal entry using some other class." + - " This is a bug; please file an issue or email the developer mailing list. It is " + - "likely that you would not have this problem if you updated to use WALInputFormat. " + - "You will need the following exception details when seeking help from the HBase " + - "community.", - exception); - throw exception; - } - return (HLogKey)currentEntry.getKey(); - } - } - - @Override - public List getSplits(JobContext context) throws IOException, - InterruptedException { - return delegate.getSplits(context, START_TIME_KEY, END_TIME_KEY); - } - - @Override - public RecordReader createRecordReader(InputSplit split, - TaskAttemptContext context) throws IOException, InterruptedException { - return new HLogKeyRecordReader(); - } -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index 452714b..794aea2 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -80,8 +80,6 @@ public class WALPlayer extends Configured implements Tool { Configuration.addDeprecation("hlog.bulk.output", BULK_OUTPUT_CONF_KEY); Configuration.addDeprecation("hlog.input.tables", TABLES_KEY); Configuration.addDeprecation("hlog.input.tablesmap", TABLE_MAP_KEY); - Configuration.addDeprecation(HLogInputFormat.START_TIME_KEY, WALInputFormat.START_TIME_KEY); - Configuration.addDeprecation(HLogInputFormat.END_TIME_KEY, WALInputFormat.END_TIME_KEY); } private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; @@ -244,8 +242,8 @@ public class WALPlayer extends Configured implements Tool { public Job createSubmittableJob(String[] args) throws IOException { Configuration conf = getConf(); - setupTime(conf, HLogInputFormat.START_TIME_KEY); - setupTime(conf, HLogInputFormat.END_TIME_KEY); + setupTime(conf, WALInputFormat.START_TIME_KEY); + setupTime(conf, WALInputFormat.END_TIME_KEY); Path inputDir = new Path(args[0]); String[] tables = args[1].split(","); String[] tableMap; diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 86c02ea..eaf612a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -164,8 +164,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.regionserver.wal.ReplayHLogKey; +import org.apache.hadoop.hbase.regionserver.wal.ReplayWALKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.regionserver.wal.WALUtil; import org.apache.hadoop.hbase.security.User; @@ -3223,7 +3222,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi WALKey walKey = null; if (replay) { // use wal key from the original - walKey = new ReplayHLogKey(this.getRegionInfo().getEncodedNameAsBytes(), + walKey = new ReplayWALKey(this.getRegionInfo().getEncodedNameAsBytes(), this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now, mutation.getClusterIds(), currentNonceGroup, currentNonce, mvcc); walKey.setOrigLogSeqNum(batchOp.getReplaySequenceId()); @@ -3232,8 +3231,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // called for replayed edits? Am afraid to change it without test. if (!walEdit.isEmpty()) { if (!replay) { - // we use HLogKey here instead of WALKey directly to support legacy coprocessors. - walKey = new HLogKey(this.getRegionInfo().getEncodedNameAsBytes(), + walKey = new WALKey(this.getRegionInfo().getEncodedNameAsBytes(), this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now, mutation.getClusterIds(), currentNonceGroup, currentNonce, mvcc, this.getReplicationScope()); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index 567664e..b630b30 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -78,7 +78,6 @@ import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.regionserver.Region.Operation; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -104,7 +103,7 @@ public class RegionCoprocessorHost private final boolean hasCustomPostScannerFilterRow; /** - * + * * Encapsulation of the environment of each coprocessor */ static class RegionEnvironment extends CoprocessorHost.Environment @@ -113,8 +112,6 @@ public class RegionCoprocessorHost private Region region; private RegionServerServices rsServices; ConcurrentMap sharedData; - private final boolean useLegacyPre; - private final boolean useLegacyPost; /** * Constructor @@ -128,14 +125,6 @@ public class RegionCoprocessorHost this.region = region; this.rsServices = services; this.sharedData = sharedData; - // Pick which version of the WAL related events we'll call. - // This way we avoid calling the new version on older RegionObservers so - // we can maintain binary compatibility. - // See notes in javadoc for RegionObserver - useLegacyPre = useLegacyMethod(impl.getClass(), "preWALRestore", ObserverContext.class, - HRegionInfo.class, WALKey.class, WALEdit.class); - useLegacyPost = useLegacyMethod(impl.getClass(), "postWALRestore", ObserverContext.class, - HRegionInfo.class, WALKey.class, WALEdit.class); } /** @return the region */ @@ -150,6 +139,7 @@ public class RegionCoprocessorHost return rsServices; } + @Override public void shutdown() { super.shutdown(); } @@ -369,7 +359,7 @@ public class RegionCoprocessorHost // scan the table attributes for coprocessor load specifications // initialize the coprocessors List configured = new ArrayList(); - for (TableCoprocessorAttribute attr: getTableCoprocessorAttrsFromSchema(conf, + for (TableCoprocessorAttribute attr: getTableCoprocessorAttrsFromSchema(conf, region.getTableDesc())) { // Load encompasses classloading and coprocessor initialization try { @@ -520,6 +510,7 @@ public class RegionCoprocessorHost throws IOException { oserver.postClose(ctx, abortRequested); } + @Override public void postEnvCall(RegionEnvironment env) { shutdown(env); } @@ -1448,33 +1439,12 @@ public class RegionCoprocessorHost @Override public void call(RegionObserver oserver, ObserverContext ctx) throws IOException { - // Once we don't need to support the legacy call, replace RegionOperation with a version - // that's ObserverContext and avoid this cast. - final RegionEnvironment env = (RegionEnvironment)ctx.getEnvironment(); - if (env.useLegacyPre) { - if (logKey instanceof HLogKey) { - oserver.preWALRestore(ctx, info, (HLogKey)logKey, logEdit); - } else { - legacyWarning(oserver.getClass(), "There are wal keys present that are not HLogKey."); - } - } else { - oserver.preWALRestore(ctx, info, logKey, logEdit); - } + oserver.preWALRestore(ctx, info, logKey, logEdit); } }); } /** - * @return true if default behavior should be bypassed, false otherwise - * @deprecated use {@link #preWALRestore(HRegionInfo, WALKey, WALEdit)}; as of 2.0, remove in 3.0 - */ - @Deprecated - public boolean preWALRestore(final HRegionInfo info, final HLogKey logKey, - final WALEdit logEdit) throws IOException { - return preWALRestore(info, (WALKey)logKey, logEdit); - } - - /** * @param info * @param logKey * @param logEdit @@ -1486,32 +1456,12 @@ public class RegionCoprocessorHost @Override public void call(RegionObserver oserver, ObserverContext ctx) throws IOException { - // Once we don't need to support the legacy call, replace RegionOperation with a version - // that's ObserverContext and avoid this cast. - final RegionEnvironment env = (RegionEnvironment)ctx.getEnvironment(); - if (env.useLegacyPost) { - if (logKey instanceof HLogKey) { - oserver.postWALRestore(ctx, info, (HLogKey)logKey, logEdit); - } else { - legacyWarning(oserver.getClass(), "There are wal keys present that are not HLogKey."); - } - } else { - oserver.postWALRestore(ctx, info, logKey, logEdit); - } + oserver.postWALRestore(ctx, info, logKey, logEdit); } }); } /** - * @deprecated use {@link #postWALRestore(HRegionInfo, WALKey, WALEdit)}; as of 2.0, remove in 3.0 - */ - @Deprecated - public void postWALRestore(final HRegionInfo info, final HLogKey logKey, final WALEdit logEdit) - throws IOException { - postWALRestore(info, (WALKey)logKey, logEdit); - } - - /** * @param familyPaths pairs of { CF, file path } submitted for bulk load * @return true if the default operation should be bypassed * @throws IOException @@ -1685,10 +1635,12 @@ public class RegionCoprocessorHost public abstract void call(RegionObserver observer, ObserverContext ctx) throws IOException; + @Override public boolean hasCall(Coprocessor observer) { return observer instanceof RegionObserver; } + @Override public void call(Coprocessor observer, ObserverContext ctx) throws IOException { call((RegionObserver)observer, ctx); @@ -1712,10 +1664,12 @@ public class RegionCoprocessorHost public abstract void call(EndpointObserver observer, ObserverContext ctx) throws IOException; + @Override public boolean hasCall(Coprocessor observer) { return observer instanceof EndpointObserver; } + @Override public void call(Coprocessor observer, ObserverContext ctx) throws IOException { call((EndpointObserver)observer, ctx); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index f93537d..df430a4 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -589,6 +589,7 @@ public class FSHLog extends AbstractFSWAL { return sequence; } + @Override public void run() { long currentSequence; while (!isInterrupted()) { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java deleted file mode 100644 index d7bf4a5..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java +++ /dev/null @@ -1,285 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver.wal; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.EOFException; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.NavigableMap; -import java.util.UUID; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; - -import com.google.common.annotations.VisibleForTesting; - -/** - * A Key for an entry in the change log. - * - * The log intermingles edits to many tables and rows, so each log entry - * identifies the appropriate table and row. Within a table and row, they're - * also sorted. - * - *

Some Transactional edits (START, COMMIT, ABORT) will not have an - * associated row. - * @deprecated use WALKey. Deprecated as of 1.0 (HBASE-12522). Remove in 2.0 - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION) -@Deprecated -public class HLogKey extends WALKey implements Writable { - private static final Log LOG = LogFactory.getLog(HLogKey.class); - - public HLogKey() { - super(); - } - - @VisibleForTesting - public HLogKey(final byte[] encodedRegionName, final TableName tablename, long logSeqNum, - final long now, UUID clusterId) { - super(encodedRegionName, tablename, logSeqNum, now, clusterId); - } - - public HLogKey(final byte[] encodedRegionName, final TableName tablename) { - super(encodedRegionName, tablename, null); - } - - @VisibleForTesting - public HLogKey(final byte[] encodedRegionName, final TableName tablename, final long now) { - super(encodedRegionName, tablename, now); - } - - @VisibleForTesting - public HLogKey(final byte[] encodedRegionName, final TableName tablename, final long now, - final NavigableMap replicationScope) { - super(encodedRegionName, tablename, now, replicationScope); - } - - public HLogKey(final byte[] encodedRegionName, final TableName tablename, final long now, - final MultiVersionConcurrencyControl mvcc, final NavigableMap scopes) { - super(encodedRegionName, tablename, now, mvcc, scopes); - } - - /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - *

Used by log splitting and snapshots. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - name of table - * @param logSeqNum - log sequence number - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) - */ - public HLogKey( - final byte[] encodedRegionName, - final TableName tablename, - long logSeqNum, - final long now, - List clusterIds, - long nonceGroup, - long nonce, - MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc); - } - - /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - *

Used by log splitting and snapshots. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - name of table - * @param logSeqNum - log sequence number - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) - * @param nonceGroup the noncegroup - * @param nonce the nonce - * @param replicationScope the replicationScope of the non-default column families' of the region - */ - public HLogKey( - final byte[] encodedRegionName, - final TableName tablename, - long logSeqNum, - final long now, - List clusterIds, - long nonceGroup, - long nonce, - MultiVersionConcurrencyControl mvcc, NavigableMap replicationScope) { - super(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc, - replicationScope); - } - - /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - * @param now Time at which this edit was written. - * @param clusterIds the clusters that have consumed the change(used in Replication) - * @param nonceGroup - * @param nonce - */ - public HLogKey(final byte[] encodedRegionName, - final TableName tablename, - final long now, - List clusterIds, - long nonceGroup, - long nonce, - final MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, now, clusterIds, nonceGroup, nonce, mvcc); - } - - /** - * Create the log key for writing to somewhere. - * We maintain the tablename mainly for debugging purposes. - * A regionName is always a sub-table object. - * - * @param encodedRegionName Encoded name of the region as returned by - * HRegionInfo#getEncodedNameAsBytes(). - * @param tablename - * @param logSeqNum - * @param nonceGroup - * @param nonce - */ - public HLogKey(final byte [] encodedRegionName, final TableName tablename, long logSeqNum, - long nonceGroup, long nonce, MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, logSeqNum, nonceGroup, nonce, mvcc); - } - - /** - * @deprecated Don't use these Writables methods. Use PB instead. - */ - @Override - @Deprecated - public void write(DataOutput out) throws IOException { - LOG.warn("HLogKey is being serialized to writable - only expected in test code"); - WritableUtils.writeVInt(out, VERSION.code); - if (compressionContext == null) { - Bytes.writeByteArray(out, this.encodedRegionName); - Bytes.writeByteArray(out, this.tablename.getName()); - } else { - Compressor.writeCompressed(this.encodedRegionName, 0, - this.encodedRegionName.length, out, - compressionContext.regionDict); - Compressor.writeCompressed(this.tablename.getName(), 0, - this.tablename.getName().length, out, - compressionContext.tableDict); - } - out.writeLong(getSequenceId()); - out.writeLong(this.writeTime); - // Don't need to write the clusters information as we are using protobufs from 0.95 - // Writing only the first clusterId for testing the legacy read - Iterator iterator = clusterIds.iterator(); - if(iterator.hasNext()){ - out.writeBoolean(true); - UUID clusterId = iterator.next(); - out.writeLong(clusterId.getMostSignificantBits()); - out.writeLong(clusterId.getLeastSignificantBits()); - } else { - out.writeBoolean(false); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - Version version = Version.UNVERSIONED; - // HLogKey was not versioned in the beginning. - // In order to introduce it now, we make use of the fact - // that encodedRegionName was written with Bytes.writeByteArray, - // which encodes the array length as a vint which is >= 0. - // Hence if the vint is >= 0 we have an old version and the vint - // encodes the length of encodedRegionName. - // If < 0 we just read the version and the next vint is the length. - // @see Bytes#readByteArray(DataInput) - serializeReplicationScope(false); // writable HLogKey does not contain scopes - int len = WritableUtils.readVInt(in); - byte[] tablenameBytes = null; - if (len < 0) { - // what we just read was the version - version = Version.fromCode(len); - // We only compress V2 of WALkey. - // If compression is on, the length is handled by the dictionary - if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) { - len = WritableUtils.readVInt(in); - } - } - if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) { - this.encodedRegionName = new byte[len]; - in.readFully(this.encodedRegionName); - tablenameBytes = Bytes.readByteArray(in); - } else { - this.encodedRegionName = Compressor.readCompressed(in, compressionContext.regionDict); - tablenameBytes = Compressor.readCompressed(in, compressionContext.tableDict); - } - - setSequenceId(in.readLong()); - this.writeTime = in.readLong(); - - this.clusterIds.clear(); - if (version.atLeast(Version.INITIAL)) { - if (in.readBoolean()) { - // read the older log - // Definitely is the originating cluster - clusterIds.add(new UUID(in.readLong(), in.readLong())); - } - } else { - try { - // dummy read (former byte cluster id) - in.readByte(); - } catch(EOFException e) { - // Means it's a very old key, just continue - if (LOG.isTraceEnabled()) LOG.trace(e); - } - } - try { - this.tablename = TableName.valueOf(tablenameBytes); - } catch (IllegalArgumentException iae) { - if (Bytes.toString(tablenameBytes).equals(TableName.OLD_META_STR)) { - // It is a pre-namespace meta table edit, continue with new format. - LOG.info("Got an old .META. edit, continuing with new format "); - this.tablename = TableName.META_TABLE_NAME; - this.encodedRegionName = HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); - } else if (Bytes.toString(tablenameBytes).equals(TableName.OLD_ROOT_STR)) { - this.tablename = TableName.OLD_ROOT_TABLE_NAME; - throw iae; - } else throw iae; - } - // Do not need to read the clusters information as we are using protobufs from 0.95 - } - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java index ed51db5..b34c7f0 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL.Entry; +import org.apache.hadoop.hbase.wal.WALKey; @InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.PHOENIX}) public abstract class ReaderBase implements AbstractFSWALProvider.Reader { @@ -91,9 +92,7 @@ public abstract class ReaderBase implements AbstractFSWALProvider.Reader { public Entry next(Entry reuse) throws IOException { Entry e = reuse; if (e == null) { - // we use HLogKey here instead of WALKey directly to support legacy coprocessors, - // seqencefile based readers, and HLogInputFormat. - e = new Entry(new HLogKey(), new WALEdit()); + e = new Entry(new WALKey(), new WALEdit()); } if (compressionContext != null) { e.setCompressionContext(compressionContext); @@ -147,7 +146,7 @@ public abstract class ReaderBase implements AbstractFSWALProvider.Reader { * Initializes the compression after the shared stuff has been initialized. Called once. */ protected abstract void initAfterCompression() throws IOException; - + /** * Initializes the compression after the shared stuff has been initialized. Called once. * @param cellCodecClsName class name of cell Codec diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayHLogKey.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayHLogKey.java deleted file mode 100644 index d5a1561..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayHLogKey.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver.wal; - -import java.util.List; -import java.util.UUID; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; - -/** - * An HLogKey specific to WalEdits coming from replay. - */ -@InterfaceAudience.Private -public class ReplayHLogKey extends HLogKey { - - public ReplayHLogKey(final byte [] encodedRegionName, final TableName tablename, - final long now, List clusterIds, long nonceGroup, long nonce, - MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, now, clusterIds, nonceGroup, nonce, mvcc); - } - - public ReplayHLogKey(final byte [] encodedRegionName, final TableName tablename, - long logSeqNum, final long now, List clusterIds, long nonceGroup, long nonce, - MultiVersionConcurrencyControl mvcc) { - super(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc); - } - - /** - * Returns the original sequence id - * @return long the new assigned sequence number - */ - @Override - public long getSequenceId() { - return this.getOrigLogSeqNum(); - } -} \ No newline at end of file diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayWALKey.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayWALKey.java new file mode 100644 index 0000000..3bff78f --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReplayWALKey.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.util.List; +import java.util.UUID; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; +import org.apache.hadoop.hbase.wal.WALKey; + +/** + * An HLogKey specific to WalEdits coming from replay. + */ +@InterfaceAudience.Private +public class ReplayWALKey extends WALKey { + + public ReplayWALKey(final byte [] encodedRegionName, final TableName tablename, + final long now, List clusterIds, long nonceGroup, long nonce, + MultiVersionConcurrencyControl mvcc) { + super(encodedRegionName, tablename, now, clusterIds, nonceGroup, nonce, mvcc); + } + + public ReplayWALKey(final byte [] encodedRegionName, final TableName tablename, + long logSeqNum, final long now, List clusterIds, long nonceGroup, long nonce, + MultiVersionConcurrencyControl mvcc) { + super(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc); + } + + /** + * Returns the original sequence id + * @return long the new assigned sequence number + */ + @Override + public long getSequenceId() { + return this.getOrigLogSeqNum(); + } +} \ No newline at end of file diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java index 87019e8..4c02ee3 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java @@ -48,9 +48,6 @@ public class WALCoprocessorHost private final WAL wal; - final boolean useLegacyPre; - final boolean useLegacyPost; - @Override public WAL getWAL() { return wal; @@ -70,14 +67,6 @@ public class WALCoprocessorHost final WAL wal) { super(impl, priority, seq, conf); this.wal = wal; - // Pick which version of the API we'll call. - // This way we avoid calling the new version on older WALObservers so - // we can maintain binary compatibility. - // See notes in javadoc for WALObserver - useLegacyPre = useLegacyMethod(impl.getClass(), "preWALWrite", ObserverContext.class, - HRegionInfo.class, WALKey.class, WALEdit.class); - useLegacyPost = useLegacyMethod(impl.getClass(), "postWALWrite", ObserverContext.class, - HRegionInfo.class, WALKey.class, WALEdit.class); } } @@ -131,16 +120,7 @@ public class WALCoprocessorHost ClassLoader cl = currentThread.getContextClassLoader(); try { currentThread.setContextClassLoader(env.getClassLoader()); - if (env.useLegacyPre) { - if (logKey instanceof HLogKey) { - observer.preWALWrite(ctx, info, (HLogKey)logKey, logEdit); - } else { - legacyWarning(observer.getClass(), - "There are wal keys present that are not HLogKey."); - } - } else { - observer.preWALWrite(ctx, info, logKey, logEdit); - } + observer.preWALWrite(ctx, info, logKey, logEdit); } catch (Throwable e) { handleCoprocessorThrowable(env, e); } finally { @@ -175,16 +155,7 @@ public class WALCoprocessorHost ClassLoader cl = currentThread.getContextClassLoader(); try { currentThread.setContextClassLoader(env.getClassLoader()); - if (env.useLegacyPost) { - if (logKey instanceof HLogKey) { - observer.postWALWrite(ctx, info, (HLogKey)logKey, logEdit); - } else { - legacyWarning(observer.getClass(), - "There are wal keys present that are not HLogKey."); - } - } else { - observer.postWALWrite(ctx, info, logKey, logEdit); - } + observer.postWALWrite(ctx, info, logKey, logEdit); } catch (Throwable e) { handleCoprocessorThrowable(env, e); } finally { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java index 2d98919..293669b 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java @@ -610,6 +610,8 @@ public class AccessControlLists { throw new DeserializationException(e); } } else { + // TODO: We have to re-write non-PB data as PB encoded. Otherwise we will carry old Writables + // forever (here and a couple of other places). ListMultimap perms = ArrayListMultimap.create(); try { DataInput in = new DataInputStream(new ByteArrayInputStream(data)); @@ -674,7 +676,7 @@ public class AccessControlLists { // Deserialize the table permissions from the KV // TODO: This can be improved. Don't build UsersAndPermissions just to unpack it again, // use the builder - AccessControlProtos.UsersAndPermissions.Builder builder = + AccessControlProtos.UsersAndPermissions.Builder builder = AccessControlProtos.UsersAndPermissions.newBuilder(); if (tag.hasArray()) { ProtobufUtil.mergeFrom(builder, tag.getValueArray(), tag.getValueOffset(), diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java index d0d9b63..dc9988d 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HbaseObjectWritableFor96Migration.java @@ -86,10 +86,7 @@ import org.apache.hadoop.hbase.io.WritableWithSize; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.regionserver.RegionOpeningState; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.wal.WAL.Entry; -import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ProtoUtil; import org.apache.hadoop.io.DataOutputOutputStream; @@ -232,7 +229,7 @@ class HbaseObjectWritableFor96Migration implements Writable, WritableWithSize, C addToMap(Entry.class, code++); addToMap(Entry[].class, code++); - addToMap(HLogKey.class, code++); + code++; // HLogKey is removed addToMap(List.class, code++); @@ -334,14 +331,17 @@ class HbaseObjectWritableFor96Migration implements Writable, WritableWithSize, C } + @Override public void readFields(DataInput in) throws IOException { readObject(in, this, this.conf); } + @Override public void write(DataOutput out) throws IOException { writeObject(out, instance, declaredClass, conf); } + @Override public long getWritableSize() { return getWritableSize(instance, declaredClass, conf); } @@ -361,10 +361,12 @@ class HbaseObjectWritableFor96Migration implements Writable, WritableWithSize, C this.declaredClass = declaredClass; } + @Override public void readFields(DataInput in) throws IOException { this.declaredClass = CODE_TO_CLASS.get(WritableUtils.readVInt(in)); } + @Override public void write(DataOutput out) throws IOException { writeClassCode(out, this.declaredClass); } @@ -392,7 +394,7 @@ class HbaseObjectWritableFor96Migration implements Writable, WritableWithSize, C } /** - * @return the next object code in the list. Used in testing to verify that additional fields are not added + * @return the next object code in the list. Used in testing to verify that additional fields are not added */ static int getNextClassCode(){ return NEXT_CLASS_CODE; @@ -541,26 +543,6 @@ class HbaseObjectWritableFor96Migration implements Writable, WritableWithSize, C byte [] scanBytes = ProtobufUtil.toScan(scan).toByteArray(); out.writeInt(scanBytes.length); out.write(scanBytes); - } else if (Entry.class.isAssignableFrom(declClass)) { - // Entry is no longer Writable, maintain compatible serialization. - // Writables write their exact runtime class - Class c = instanceObj.getClass(); - Integer code = CLASS_TO_CODE.get(c); - if (code == null) { - out.writeByte(NOT_ENCODED); - Text.writeString(out, c.getName()); - } else { - writeClassCode(out, c); - } - final Entry entry = (Entry)instanceObj; - // We only support legacy HLogKey - WALKey key = entry.getKey(); - if (!(key instanceof HLogKey)) { - throw new IOException("Can't write Entry '" + instanceObj + "' due to key class '" + - key.getClass() + "'"); - } - ((HLogKey)key).write(out); - entry.getEdit().write(out); } else { throw new IOException("Can't write: "+instanceObj+" as "+declClass); } @@ -721,13 +703,6 @@ class HbaseObjectWritableFor96Migration implements Writable, WritableWithSize, C declaredClass = ((NullInstance)instance).declaredClass; instance = null; } - } else if (Entry.class.isAssignableFrom(instanceClass)) { - // Entry stopped being Writable; maintain serialization support. - final HLogKey key = new HLogKey(); - final WALEdit edit = new WALEdit(); - key.readFields(in); - edit.readFields(in); - instance = new Entry(key, edit); } else { int length = in.readInt(); byte[] objectBytes = new byte[length]; @@ -834,10 +809,12 @@ class HbaseObjectWritableFor96Migration implements Writable, WritableWithSize, C CODE_TO_CLASS.put(code, clazz); } + @Override public void setConf(Configuration conf) { this.conf = conf; } + @Override public Configuration getConf() { return this.conf; } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java index 25cfc8b..72c3fed 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java @@ -164,7 +164,7 @@ public class TableAuthManager implements Closeable { return this.zkperms; } - public void refreshTableCacheFromWritable(TableName table, + public void refreshTableCacheFromSerializedData(TableName table, byte[] data) throws IOException { if (data != null && data.length > 0) { ListMultimap perms; @@ -186,7 +186,7 @@ public class TableAuthManager implements Closeable { } } - public void refreshNamespaceCacheFromWritable(String namespace, byte[] data) throws IOException { + public void refreshNamespaceCacheFromSerializedData(String namespace, byte[] data) throws IOException { if (data != null && data.length > 0) { ListMultimap perms; try { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java index c4b3219..c581968 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java @@ -235,10 +235,10 @@ public class ZKPermissionWatcher extends ZooKeeperListener implements Closeable Bytes.toStringBinary(nodeData)); } if(AccessControlLists.isNamespaceEntry(entry)) { - authManager.refreshNamespaceCacheFromWritable( + authManager.refreshNamespaceCacheFromSerializedData( AccessControlLists.fromNamespaceEntry(entry), nodeData); } else { - authManager.refreshTableCacheFromWritable(TableName.valueOf(entry), nodeData); + authManager.refreshTableCacheFromSerializedData(TableName.valueOf(entry), nodeData); } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java index 3d74a97..93b0045 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; // imports we use from yet-to-be-moved regionsever.wal import org.apache.hadoop.hbase.regionserver.wal.CompressionContext; import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -242,8 +241,7 @@ public interface WAL { public Entry() { edit = new WALEdit(); - // we use HLogKey here instead of WALKey directly to support legacy coprocessors. - key = new HLogKey(); + key = new WALKey(); } /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java index 86fdfbd..fa7dc85 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java @@ -59,8 +59,6 @@ import com.google.protobuf.ByteString; * identifies the appropriate table and row. Within a table and row, they're * also sorted. * - *

Some Transactional edits (START, COMMIT, ABORT) will not have an associated row. - * * Note that protected members marked @InterfaceAudience.Private are only protected * to support the legacy HLogKey class, which is in a different package. */ @@ -220,21 +218,15 @@ public class WALKey implements SequenceId, Comparable { @VisibleForTesting public WALKey(final byte[] encodedRegionName, final TableName tablename, - long logSeqNum, - final long now, UUID clusterId) { + long logSeqNum, final long now, UUID clusterId) { List clusterIds = new ArrayList(); clusterIds.add(clusterId); init(encodedRegionName, tablename, logSeqNum, now, clusterIds, HConstants.NO_NONCE, HConstants.NO_NONCE, null, null); } - /** - * @deprecated Remove. Useless. - */ - @Deprecated // REMOVE - public WALKey(final byte[] encodedRegionName, final TableName tablename, - final NavigableMap replicationScope) { - this(encodedRegionName, tablename, System.currentTimeMillis(), replicationScope); + public WALKey(final byte[] encodedRegionName, final TableName tablename) { + this(encodedRegionName, tablename, EnvironmentEdgeManager.currentTime()); } // TODO: Fix being able to pass in sequenceid. @@ -461,7 +453,7 @@ public class WALKey implements SequenceId, Comparable { public void setOrigLogSeqNum(final long sequenceId) { this.origLogSeqNum = sequenceId; } - + /** * Return a positive long if current WALKey is created from a replay edit; a replay edit is an * edit that came in when replaying WALs of a crashed server. @@ -470,7 +462,7 @@ public class WALKey implements SequenceId, Comparable { public long getOrigLogSeqNum() { return this.origLogSeqNum; } - + /** * SequenceId is only available post WAL-assign. Calls before this will get you a * {@link #NO_SEQUENCE_ID}. See the comment on FSHLog#append and #getWriteNumber in this method diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index beccb04..f0ae6a7 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -106,7 +106,6 @@ import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.R import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.LastSequenceId; import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.regionserver.wal.WALEditsReplaySink; @@ -2364,8 +2363,7 @@ public class WALSplitter { for (HBaseProtos.UUID uuid : entry.getKey().getClusterIdsList()) { clusterIds.add(new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits())); } - // we use HLogKey here instead of WALKey directly to support legacy coprocessors. - key = new HLogKey(walKeyProto.getEncodedRegionName().toByteArray(), TableName.valueOf( + key = new WALKey(walKeyProto.getEncodedRegionName().toByteArray(), TableName.valueOf( walKeyProto.getTableName().toByteArray()), replaySeqId, walKeyProto.getWriteTime(), clusterIds, walKeyProto.getNonceGroup(), walKeyProto.getNonce(), null); logEntry.setFirst(key); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java index ebd6d8e..63b3645 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.wal.WALKey; @@ -103,13 +102,6 @@ implements WALObserver { } @Override - public void postWALWrite(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - postWALWriteDeprecatedCalled = true; - postWALWrite(env, info, (WALKey)logKey, logEdit); - } - - @Override public boolean preWALWrite(ObserverContext env, HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { boolean bypass = false; @@ -148,13 +140,6 @@ implements WALObserver { return bypass; } - @Override - public boolean preWALWrite(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - preWALWriteDeprecatedCalled = true; - return preWALWrite(env, info, (WALKey)logKey, logEdit); - } - /** * Triggered before {@link org.apache.hadoop.hbase.regionserver.HRegion} when WAL is * Restoreed. @@ -166,13 +151,6 @@ implements WALObserver { } @Override - public void preWALRestore(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - preWALRestoreDeprecatedCalled = true; - preWALRestore(env, info, (WALKey)logKey, logEdit); - } - - @Override public void preWALRoll(ObserverContext ctx, Path oldPath, Path newPath) throws IOException { preWALRollCalled = true; @@ -194,13 +172,6 @@ implements WALObserver { postWALRestoreCalled = true; } - @Override - public void postWALRestore(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - postWALRestoreDeprecatedCalled = true; - postWALRestore(env, info, (WALKey)logKey, logEdit); - } - public boolean isPreWALWriteCalled() { return preWALWriteCalled; } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java index dfd41a8..25ce9ed 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java @@ -65,7 +65,6 @@ import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileReader; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -677,26 +676,12 @@ public class SimpleRegionObserver extends BaseRegionObserver { } @Override - public void preWALRestore(ObserverContext env, HRegionInfo info, - HLogKey logKey, WALEdit logEdit) throws IOException { - preWALRestore(env, info, (WALKey)logKey, logEdit); - ctPreWALRestoreDeprecated.incrementAndGet(); - } - - @Override public void postWALRestore(ObserverContext env, HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException { ctPostWALRestore.incrementAndGet(); } @Override - public void postWALRestore(ObserverContext env, - HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - postWALRestore(env, info, (WALKey)logKey, logEdit); - ctPostWALRestoreDeprecated.incrementAndGet(); - } - - @Override public StoreFileReader preStoreFileReaderOpen(ObserverContext ctx, FileSystem fs, Path p, FSDataInputStreamWrapper in, long size, CacheConfig cacheConf, Reference r, StoreFileReader reader) throws IOException { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java index 3c591f8..29ffb31 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java @@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.security.User; @@ -239,9 +238,8 @@ public class TestWALObserver { // it's where WAL write cp should occur. long now = EnvironmentEdgeManager.currentTime(); - // we use HLogKey here instead of WALKey directly to support legacy coprocessors. long txid = log.append(hri, - new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, scopes), edit, true); + new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, scopes), edit, true); log.sync(txid); // the edit shall have been change now by the coprocessor. @@ -271,86 +269,6 @@ public class TestWALObserver { assertEquals(seesLegacy, cp.isPostWALWriteDeprecatedCalled()); } - @Test - public void testNonLegacyWALKeysDoNotExplode() throws Exception { - TableName tableName = TableName.valueOf(TEST_TABLE); - final HTableDescriptor htd = createBasic3FamilyHTD(Bytes - .toString(TEST_TABLE)); - final HRegionInfo hri = new HRegionInfo(tableName, null, null); - MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); - - fs.mkdirs(new Path(FSUtils.getTableDir(hbaseRootDir, tableName), hri.getEncodedName())); - - final Configuration newConf = HBaseConfiguration.create(this.conf); - - final WAL wal = wals.getWAL(UNSPECIFIED_REGION, null); - final SampleRegionWALObserver newApi = getCoprocessor(wal, SampleRegionWALObserver.class); - newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null); - final SampleRegionWALObserver oldApi = getCoprocessor(wal, - SampleRegionWALObserver.Legacy.class); - oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null); - - LOG.debug("ensuring wal entries haven't happened before we start"); - assertFalse(newApi.isPreWALWriteCalled()); - assertFalse(newApi.isPostWALWriteCalled()); - assertFalse(newApi.isPreWALWriteDeprecatedCalled()); - assertFalse(newApi.isPostWALWriteDeprecatedCalled()); - assertFalse(oldApi.isPreWALWriteCalled()); - assertFalse(oldApi.isPostWALWriteCalled()); - assertFalse(oldApi.isPreWALWriteDeprecatedCalled()); - assertFalse(oldApi.isPostWALWriteDeprecatedCalled()); - - LOG.debug("writing to WAL with non-legacy keys."); - NavigableMap scopes = new TreeMap( - Bytes.BYTES_COMPARATOR); - for (HColumnDescriptor hcd : htd.getFamilies()) { - scopes.put(hcd.getName(), 0); - } - final int countPerFamily = 5; - for (HColumnDescriptor hcd : htd.getFamilies()) { - addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, - EnvironmentEdgeManager.getDelegate(), wal, scopes, mvcc); - } - - LOG.debug("Verify that only the non-legacy CP saw edits."); - assertTrue(newApi.isPreWALWriteCalled()); - assertTrue(newApi.isPostWALWriteCalled()); - assertFalse(newApi.isPreWALWriteDeprecatedCalled()); - assertFalse(newApi.isPostWALWriteDeprecatedCalled()); - // wish we could test that the log message happened :/ - assertFalse(oldApi.isPreWALWriteCalled()); - assertFalse(oldApi.isPostWALWriteCalled()); - assertFalse(oldApi.isPreWALWriteDeprecatedCalled()); - assertFalse(oldApi.isPostWALWriteDeprecatedCalled()); - - LOG.debug("reseting cp state."); - newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null); - oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null); - - LOG.debug("write a log edit that supports legacy cps."); - final long now = EnvironmentEdgeManager.currentTime(); - final WALKey legacyKey = new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), now); - final WALEdit edit = new WALEdit(); - final byte[] nonce = Bytes.toBytes("1772"); - edit.add(new KeyValue(TEST_ROW, TEST_FAMILY[0], nonce, now, nonce)); - final long txid = wal.append(hri, legacyKey, edit, true); - wal.sync(txid); - - LOG.debug("Make sure legacy cps can see supported edits after having been skipped."); - assertTrue("non-legacy WALObserver didn't see pre-write.", newApi.isPreWALWriteCalled()); - assertTrue("non-legacy WALObserver didn't see post-write.", newApi.isPostWALWriteCalled()); - assertFalse("non-legacy WALObserver shouldn't have seen legacy pre-write.", - newApi.isPreWALWriteDeprecatedCalled()); - assertFalse("non-legacy WALObserver shouldn't have seen legacy post-write.", - newApi.isPostWALWriteDeprecatedCalled()); - assertTrue("legacy WALObserver didn't see pre-write.", oldApi.isPreWALWriteCalled()); - assertTrue("legacy WALObserver didn't see post-write.", oldApi.isPostWALWriteCalled()); - assertTrue("legacy WALObserver didn't see legacy pre-write.", - oldApi.isPreWALWriteDeprecatedCalled()); - assertTrue("legacy WALObserver didn't see legacy post-write.", - oldApi.isPostWALWriteDeprecatedCalled()); - } - /** * Coprocessors shouldn't get notice of empty waledits. */ @@ -432,6 +350,7 @@ public class TestWALObserver { User user = HBaseTestingUtility.getDifferentUser(newConf, ".replay.wal.secondtime"); user.runAs(new PrivilegedExceptionAction() { + @Override public Object run() throws Exception { Path p = runWALSplit(newConf); LOG.info("WALSplit path == " + p); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java deleted file mode 100644 index 752faa6..0000000 --- hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.mapreduce; - -import org.apache.hadoop.hbase.mapreduce.WALInputFormat.WALRecordReader; - -import java.util.NavigableMap; - -import org.apache.hadoop.hbase.mapreduce.HLogInputFormat.HLogKeyRecordReader; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; -import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.experimental.categories.Category; - -/** - * JUnit tests for the record reader in HLogInputFormat - */ -@Category({MapReduceTests.class, MediumTests.class}) -public class TestHLogRecordReader extends TestWALRecordReader { - - @Override - protected WALKey getWalKey(final long time, NavigableMap scopes) { - return new HLogKey(info.getEncodedNameAsBytes(), tableName, time, mvcc, scopes); - } - - @Override - protected WALRecordReader getReader() { - return new HLogKeyRecordReader(); - } -} diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index 002438a..360dda0 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -90,7 +90,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -101,9 +100,9 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; -import org.apache.hadoop.hbase.wal.FSHLogProvider; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; +import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -1301,8 +1300,8 @@ public class TestDistributedLogSplitting { WALEdit e = new WALEdit(); value++; e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); - wal.append(curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(), tableName, - System.currentTimeMillis(), null), e, true); + wal.append(curRegionInfo, new WALKey(curRegionInfo.getEncodedNameAsBytes(), tableName, + System.currentTimeMillis()), e, true); } wal.sync(); wal.shutdown(); @@ -1396,7 +1395,7 @@ public class TestDistributedLogSplitting { WALEdit e = new WALEdit(); value++; e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); - wal.append(curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(), + wal.append(curRegionInfo, new WALKey(curRegionInfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis()), e, true); } wal.sync(); @@ -1610,7 +1609,7 @@ public class TestDistributedLogSplitting { byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i)); e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value)); log.append(curRegionInfo, - new HLogKey(curRegionInfo.getEncodedNameAsBytes(), fullTName, + new WALKey(curRegionInfo.getEncodedNameAsBytes(), fullTName, System.currentTimeMillis()), e, true); if (0 == i % syncEvery) { log.sync(); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 765c9cf..7d55635 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -90,7 +90,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TestMobSnapshotCloneIndependence; import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; @@ -103,7 +102,6 @@ import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.master.procedure.TestMasterFailoverWithProcedures; import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; @@ -119,7 +117,6 @@ import org.apache.hadoop.hbase.regionserver.Region.RowLock; import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem; import org.apache.hadoop.hbase.regionserver.handler.FinishRegionRecoveringHandler; import org.apache.hadoop.hbase.regionserver.wal.FSHLog; -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; @@ -701,7 +698,7 @@ public class TestHRegion { WALEdit edit = new WALEdit(); edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); - writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time, + writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); @@ -754,7 +751,7 @@ public class TestHRegion { WALEdit edit = new WALEdit(); edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); - writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time, + writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); @@ -859,7 +856,7 @@ public class TestHRegion { edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes .toBytes(i))); } - writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time, + writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); writer.close(); } @@ -957,7 +954,7 @@ public class TestHRegion { long time = System.nanoTime(); - writer.append(new WAL.Entry(new HLogKey(regionName, tableName, 10, time, + writer.append(new WAL.Entry(new WALKey(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor))); writer.close(); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java index e9bb468..c86b6b9 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; @@ -220,7 +221,7 @@ public class TestWALLockup { Put put = new Put(bytes); put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes); WALKey key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), - scopes); + EnvironmentEdgeManager.currentTime(), scopes); WALEdit edit = new WALEdit(); CellScanner CellScanner = put.cellScanner(); assertTrue(CellScanner.advance()); @@ -247,6 +248,7 @@ public class TestWALLockup { // in HBASE-14317. Flush hangs trying to get sequenceid because the ringbuffer is held up // by the zigzaglatch waiting on syncs to come home. Thread t = new Thread ("Flusher") { + @Override public void run() { try { if (region.getMemstoreSize() <= 0) { @@ -263,7 +265,7 @@ public class TestWALLockup { }; t.setDaemon(true); t.start(); - // Wait until + // Wait until while (dodgyWAL.latch.getCount() > 0) Threads.sleep(1); // Now assert I got a new WAL file put in place even though loads of errors above. assertTrue(originalWAL != dodgyWAL.getCurrentFileName()); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java index c906d6a..f4c6cd4 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.wal.WAL.Entry; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -57,20 +58,20 @@ public class TestReplicationWALEntryFilters { SystemTableWALEntryFilter filter = new SystemTableWALEntryFilter(); // meta - WALKey key1 = new WALKey( HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), - TableName.META_TABLE_NAME, null); + WALKey key1 = new WALKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), + TableName.META_TABLE_NAME); Entry metaEntry = new Entry(key1, null); assertNull(filter.filter(metaEntry)); // ns table - WALKey key2 = new WALKey(new byte[] {}, TableName.NAMESPACE_TABLE_NAME, null); + WALKey key2 = new WALKey(new byte[] {}, TableName.NAMESPACE_TABLE_NAME); Entry nsEntry = new Entry(key2, null); assertNull(filter.filter(nsEntry)); // user table - WALKey key3 = new WALKey(new byte[] {}, TableName.valueOf("foo"), null); + WALKey key3 = new WALKey(new byte[] {}, TableName.valueOf("foo")); Entry userEntry = new Entry(key3, null); assertEquals(userEntry, filter.filter(userEntry)); @@ -237,7 +238,8 @@ public class TestReplicationWALEntryFilters { } private Entry createEntry(TreeMap scopes, byte[]... kvs) { - WALKey key1 = new WALKey(new byte[] {}, TableName.valueOf("foo"), scopes); + WALKey key1 = new WALKey(new byte[] {}, TableName.valueOf("foo"), + EnvironmentEdgeManager.currentTime(), scopes); WALEdit edit1 = new WALEdit(); for (byte[] kv : kvs) { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java deleted file mode 100644 index 15b419c..0000000 --- hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.wal; - - -import java.util.NavigableMap; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.junit.experimental.categories.Category; - -import org.apache.hadoop.hbase.regionserver.wal.HLogKey; - -@Category({RegionServerTests.class, LargeTests.class}) -public class TestDefaultWALProviderWithHLogKey extends TestFSHLogProvider { - @Override - WALKey getWalKey(final byte[] info, final TableName tableName, final long timestamp, - final NavigableMap scopes) { - return new HLogKey(info, tableName, timestamp, mvcc, scopes); - } -}