diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 6741957..583a1e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -302,7 +302,8 @@ public class HFile { try { ostream.setDropBehind(shouldDropBehind && cacheConf.shouldDropBehindCompaction()); } catch (UnsupportedOperationException uoe) { - LOG.debug("Unable to set drop behind on " + path, uoe); + if (LOG.isDebugEnabled()) LOG.debug("Unable to set drop behind on " + path); + else if (LOG.isTraceEnabled()) LOG.trace("Unable to set drop behind on " + path, uoe); } } return createWriter(fs, path, ostream, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java index 42621c9..f6ea815 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java @@ -26,19 +26,15 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.RemoteExceptionHandler; import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; -import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.HasThread; - -import java.io.IOException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; +import org.apache.hadoop.hbase.wal.WAL; import com.google.common.annotations.VisibleForTesting; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index cb2203a..2bb017b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -46,6 +46,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.io.InterruptedIOException; +import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; @@ -55,8 +56,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; +import java.util.Set; import java.util.TreeMap; import java.util.UUID; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -73,7 +76,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.DroppedSnapshotException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestCase; @@ -91,12 +96,14 @@ import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.RegionTooBusyException; +import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; @@ -107,6 +114,7 @@ import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; +import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; @@ -118,20 +126,25 @@ import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.ipc.RpcServerInterface; +import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor; +import org.apache.hadoop.hbase.quotas.RegionServerQuotaManager; import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; import org.apache.hadoop.hbase.regionserver.Region.RowLock; import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem; import org.apache.hadoop.hbase.regionserver.handler.FinishRegionRecoveringHandler; +import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; @@ -156,6 +169,10 @@ import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALProvider; import org.apache.hadoop.hbase.wal.WALSplitter; +import org.apache.hadoop.hbase.wal.WALProvider.Writer; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -171,6 +188,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.protobuf.ByteString; +import com.google.protobuf.Service; /** * Basic stand-alone testing of HRegion. No clusters! @@ -233,6 +251,403 @@ public class TestHRegion { } /** + * Reproduce locking up that happens when we get an IOE out of an append. See HBASE-14317. + * First I need to set up some fat mocks for Server and RegionServerServices. I also need to + * set up a WAL that will throw an exception when we go to append to it. + */ + @Test // FIX (timeout=60000) + public void testLockedUpWALSystem() throws IOException { + /** + * A WAL that we can have throw exceptions when a flag is set. + */ + class DodgyFSLog extends FSHLog { + volatile boolean throwException = false; + public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration conf) + throws IOException { + super(fs, root, logDir, conf); + } + @Override + protected Writer createWriterInstance(Path path) throws IOException { + final Writer w = super.createWriterInstance(path); + return new Writer() { + @Override + public void close() throws IOException { + w.close(); + } + + @Override + public void sync() throws IOException { + if (throwException) throw new IOException("FAKE! Failed to replace a bad datanode..."); + w.sync(); + } + + @Override + public void append(Entry entry) throws IOException { + if (throwException) throw new IOException("FAKE! Failed to replace a bad datanode..."); + w.append(entry); + } + + @Override + public long getLength() throws IOException { + return w.getLength(); + } + }; + } + } + + /** + * Mocked Server + */ + Server server = new Server () { + + @Override + public void abort(String why, Throwable e) { + // TODO Auto-generated method stub + + } + + @Override + public boolean isAborted() { + // TODO Auto-generated method stub + return false; + } + + @Override + public void stop(String why) { + // TODO Auto-generated method stub + + } + + @Override + public boolean isStopped() { + // TODO Auto-generated method stub + return false; + } + + @Override + public Configuration getConfiguration() { + return TEST_UTIL.getConfiguration(); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ClusterConnection getConnection() { + // TODO Auto-generated method stub + return null; + } + + @Override + public MetaTableLocator getMetaTableLocator() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ServerName getServerName() { + // TODO Auto-generated method stub + return null; + } + + @Override + public CoordinatedStateManager getCoordinatedStateManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ChoreService getChoreService() { + // TODO Auto-generated method stub + return null; + } + }; + /** + * Set up RegionServerServices mock. + */ + RegionServerServices services = new RegionServerServices() { + + @Override + public void addToOnlineRegions(Region r) { + // TODO Auto-generated method stub + + } + + @Override + public boolean removeFromOnlineRegions(Region r, ServerName destination) { + // TODO Auto-generated method stub + return false; + } + + @Override + public Region getFromOnlineRegions(String encodedRegionName) { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getOnlineRegions(TableName tableName) throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Configuration getConfiguration() { + return null; + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ClusterConnection getConnection() { + // TODO Auto-generated method stub + return null; + } + + @Override + public MetaTableLocator getMetaTableLocator() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ServerName getServerName() { + // TODO Auto-generated method stub + return null; + } + + @Override + public CoordinatedStateManager getCoordinatedStateManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ChoreService getChoreService() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void abort(String why, Throwable e) { + // TODO Auto-generated method stub + + } + + @Override + public boolean isAborted() { + // TODO Auto-generated method stub + return false; + } + + @Override + public void stop(String why) { + // TODO Auto-generated method stub + + } + + @Override + public boolean isStopped() { + // TODO Auto-generated method stub + return false; + } + + @Override + public void updateRegionFavoredNodesMapping(String encodedRegionName, + List favoredNodes) { + // TODO Auto-generated method stub + + } + + @Override + public InetSocketAddress[] getFavoredNodesForRegion(String encodedRegionName) { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean isStopping() { + // TODO Auto-generated method stub + return false; + } + + @Override + public WAL getWAL(HRegionInfo regionInfo) throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public CompactionRequestor getCompactionRequester() { + // TODO Auto-generated method stub + return null; + } + + @Override + public FlushRequester getFlushRequester() { + // TODO Auto-generated method stub + return null; + } + + @Override + public RegionServerAccounting getRegionServerAccounting() { + // TODO Auto-generated method stub + return null; + } + + @Override + public TableLockManager getTableLockManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public RegionServerQuotaManager getRegionServerQuotaManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void postOpenDeployTasks(PostOpenDeployContext context) + throws KeeperException, IOException { + // TODO Auto-generated method stub + + } + + @Override + public void postOpenDeployTasks(Region r) throws KeeperException, IOException { + // TODO Auto-generated method stub + + } + + @Override + public boolean reportRegionStateTransition(RegionStateTransitionContext context) { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean reportRegionStateTransition(TransitionCode code, long openSeqNum, HRegionInfo... hris) { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean reportRegionStateTransition(TransitionCode code, HRegionInfo... hris) { + // TODO Auto-generated method stub + return false; + } + + @Override + public RpcServerInterface getRpcServer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ConcurrentMap getRegionsInTransitionInRS() { + // TODO Auto-generated method stub + return null; + } + + @Override + public FileSystem getFileSystem() { + // TODO Auto-generated method stub + return null; + } + + @Override + public Leases getLeases() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ExecutorService getExecutorService() { + // TODO Auto-generated method stub + return null; + } + + @Override + public Map getRecoveringRegions() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ServerNonceManager getNonceManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean registerService(Service service) { + // TODO Auto-generated method stub + return false; + } + + @Override + public HeapMemoryManager getHeapMemoryManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public double getCompactionPressure() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public Set getOnlineTables() { + // TODO Auto-generated method stub + return null; + } + }; + // OK. Now I have my mocked up Server and RegionServerServices and my dodgy WAL, go ahead with + // the test. + FileSystem fs = FileSystem.get(CONF); + Path rootDir = new Path(dir + "testLockedUpWALSystem"); + DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, "testLockedUpWALSystem", CONF); + LogRoller logRoller = new LogRoller(server, services); + logRoller.addWAL(dodgyWAL); + logRoller.start(); + HRegion region = initHRegion(tableName, null, null, name.getMethodName(), + CONF, false, Durability.SYNC_WAL, dodgyWAL, COLUMN_FAMILY_BYTES); + try { + // Get some random bytes. + byte [] value = Bytes.toBytes(name.getMethodName()); + boolean threwIOE = false; + try { + // First get something into memstore + Put put = new Put(value); + put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), value); + region.put(put); + // Now, try and append but have it throw an IOE. + dodgyWAL.throwException = true; + put = new Put(value); + put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("2"), value); + region.put(put); + region.put(put); + } catch (IOException ioe) { + threwIOE = true; + } finally { + assertTrue("The regionserver should have thrown an exception", threwIOE); + } + } finally { + HRegion.closeHRegion(region); + if (logRoller != null) Threads.shutdown(logRoller.getThread()); + } + } + + /** * Test that I can use the max flushed sequence id after the close. * @throws IOException */ @@ -893,7 +1308,7 @@ public class TestHRegion { // now verify that the flush markers are written wal.shutdown(); - WAL.Reader reader = wals.createReader(fs, DefaultWALProvider.getCurrentFileName(wal), + WAL.Reader reader = WALFactory.createReader(fs, DefaultWALProvider.getCurrentFileName(wal), TEST_UTIL.getConfiguration()); try { List flushDescriptors = new ArrayList(); @@ -5663,7 +6078,6 @@ public class TestHRegion { putData(startRow, numRows, qualifier, families); int splitRow = startRow + numRows; putData(splitRow, numRows, qualifier, families); - int endRow = splitRow + numRows; region.flush(true); HRegion [] regions = null; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java index b3b520a..ce763b7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java @@ -414,5 +414,4 @@ public class TestFSHLog { wal.close(); } } - -} +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java index 1fcfcbb..7bf79116 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java @@ -1,4 +1,5 @@ /** + * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -44,7 +45,6 @@ public class MockServer implements Server { final ZooKeeperWatcher zk; final HBaseTestingUtility htu; - @SuppressWarnings("unused") public MockServer() throws ZooKeeperConnectionException, IOException { // Shutdown default constructor by making it private. this(null);