diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/FailedCloseWALAfterInitializedErrorException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/FailedCloseWALAfterInitializedErrorException.java new file mode 100644 index 0000000000..589004ad04 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/FailedCloseWALAfterInitializedErrorException.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase; + + +import java.io.IOException; + +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Throw when failed cleanup unsuccessful initialized wal + */ +@InterfaceAudience.Public +public class FailedCloseWALAfterInitializedErrorException + extends IOException { + + private static final long serialVersionUID = -5463156587431677322L; + + /** + * constructor with error message and throwable + * @param msg message + * @param t throwable + */ + public FailedCloseWALAfterInitializedErrorException(String msg, Throwable t) { + super(msg, t); + } + + /** + * constructor with error message + * @param msg message + */ + public FailedCloseWALAfterInitializedErrorException(String msg) { + super(msg); + } + + /** + * default constructor + */ + public FailedCloseWALAfterInitializedErrorException() { + super(); + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index fc9712b3c1..bacb58dc20 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -69,6 +69,7 @@ import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.ClockOutOfSyncException; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.FailedCloseWALAfterInitializedErrorException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; @@ -2167,11 +2168,17 @@ public class HRegionServer extends HasThread implements @Override public WAL getWAL(RegionInfo regionInfo) throws IOException { - WAL wal = walFactory.getWAL(regionInfo); - if (this.walRoller != null) { - this.walRoller.addWAL(wal); + try { + WAL wal = walFactory.getWAL(regionInfo); + if (this.walRoller != null) { + this.walRoller.addWAL(wal); + } + return wal; + }catch (FailedCloseWALAfterInitializedErrorException ex) { + // see HBASE-21751 for details + abort("wal can not clean up after init failed", ex); + throw ex; } - return wal; } public LogRoller getWalRoller() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java index 8741c1c8c6..54a91e2933 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java @@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.FailedCloseWALAfterInitializedErrorException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -157,7 +158,12 @@ public abstract class AbstractFSWALProvider> implemen succ = true; } finally { if (!succ) { - walCopy.close(); + try { + walCopy.close(); + }catch (Throwable t) { + throw new FailedCloseWALAfterInitializedErrorException( + "Failed close after init wal failed.", t); + } } } wal = walCopy; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java index 8bde6d2001..339536e28c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java @@ -480,6 +480,11 @@ public class WALFactory { return FSHLogProvider.createWriter(configuration, fs, path, false); } + @VisibleForTesting + public String getFactoryId() { + return factoryId; + } + public final WALProvider getWALProvider() { return this.provider; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALOpenError.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALOpenError.java new file mode 100644 index 0000000000..171f7bbefd --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALOpenError.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.FailedCloseWALAfterInitializedErrorException; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.wal.FSHLogProvider; +import org.apache.hadoop.hbase.wal.WALFactory; +import org.apache.hadoop.hdfs.MiniDFSCluster; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test WAL Init ERROR + */ +@Category({RegionServerTests.class, MediumTests.class}) +public class TestWALOpenError { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestWALOpenError.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestWALOpenError.class); + + protected static Configuration conf; + private static MiniDFSCluster cluster; + protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + protected static Path hbaseDir; + protected static Path hbaseWALDir; + + protected FileSystem fs; + protected Path dir; + protected WALFactory wals; + private ServerName currentServername; + + @Rule + public final TestName currentTest = new TestName(); + + @Before + public void setUp() throws Exception { + fs = cluster.getFileSystem(); + dir = new Path(hbaseDir, currentTest.getMethodName()); + this.currentServername = ServerName.valueOf(currentTest.getMethodName(), 16010, 1); + wals = new WALFactory(conf, this.currentServername.toString()); + } + + @After + public void tearDown() throws Exception { + // testAppendClose closes the FileSystem, which will prevent us from closing cleanly here. + try { + wals.close(); + } catch (IOException exception) { + LOG.warn("Encountered exception while closing wal factory. If you have other errors, this" + + " may be the cause. Message: " + exception); + LOG.debug("Exception details for failure to close wal factory.", exception); + } + FileStatus[] entries = fs.listStatus(new Path("/")); + for (FileStatus dir : entries) { + fs.delete(dir.getPath(), true); + } + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniDFSCluster(3); + conf = TEST_UTIL.getConfiguration(); + conf.set(WALFactory.WAL_PROVIDER, MyFSWalProvider.class.getName()); + conf.set(WALFactory.META_WAL_PROVIDER, MyFSWalProvider.class.getName()); + cluster = TEST_UTIL.getDFSCluster(); + + hbaseDir = TEST_UTIL.createRootDir(); + hbaseWALDir = TEST_UTIL.createWALRootDir(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + + private static MyFSLog myFSLogCreated; + private static boolean throwExceptionWhenCloseFSLogClose = false; + + @Test + public void testWALClosedIfOpenError() throws IOException { + + throwExceptionWhenCloseFSLogClose = false; + + boolean hasFakeInitException = false; + try { + wals.getWAL(HRegionInfo.FIRST_META_REGIONINFO); + } catch (IOException ex) { + hasFakeInitException = ex.getMessage().contains("Fake init exception"); + } + Assert.assertTrue(hasFakeInitException); + Assert.assertTrue(myFSLogCreated.closed); + + FileStatus[] fileStatuses = CommonFSUtils.listStatus(fs, myFSLogCreated.walDir); + Assert.assertTrue(fileStatuses == null || fileStatuses.length == 0); + } + + @Test + public void testThrowFailedCloseWalException() throws IOException { + throwExceptionWhenCloseFSLogClose = true; + boolean failedCloseWalException = false; + try { + wals.getWAL(HRegionInfo.FIRST_META_REGIONINFO); + } catch (FailedCloseWALAfterInitializedErrorException ex) { + failedCloseWalException = true; + } + Assert.assertTrue(failedCloseWalException); + } + + + public static class MyFSWalProvider extends FSHLogProvider { + + @Override + protected MyFSLog createWAL() throws IOException { + MyFSLog myFSLog = new MyFSLog(CommonFSUtils.getWALFileSystem(conf), + CommonFSUtils.getWALRootDir(conf), getWALDirectoryName(factory.getFactoryId()), + getWALArchiveDirectoryName(conf, factory.getFactoryId()), conf, listeners, true, logPrefix, + META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null); + + myFSLogCreated = myFSLog; + + return myFSLog; + } + } + + public static class MyFSLog extends FSHLog { + public MyFSLog(final FileSystem fs, final Path rootDir, final String logDir, + final String archiveDir, final Configuration conf, final List listeners, + final boolean failIfWALExists, final String prefix, final String suffix) throws IOException { + super(fs, rootDir, logDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix); + } + + @Override + public void init() throws IOException { + super.init(); + throw new IOException("Fake init exception"); + } + + @Override + public void close() throws IOException { + if (throwExceptionWhenCloseFSLogClose) { + throw new IOException("Fake close exception"); + } + super.close(); + } + } +} \ No newline at end of file