From 0d96fce0977669482592b804b62eb93fea19c339 Mon Sep 17 00:00:00 2001 From: Alex Leblang Date: Wed, 29 Nov 2017 17:11:54 -0500 Subject: [PATCH] HBASE-19369 Switch to Builder Pattern In WAL This patch makes the switch for the ProtobufLogWriter. I wanted to post my progress, but I'm still testing the code --- .../apache/hadoop/hbase/util/CommonFSUtils.java | 13 +++++ .../hbase/io/asyncfs/AsyncFSOutputHelper.java | 2 +- .../hbase/regionserver/wal/ProtobufLogWriter.java | 8 ++- .../hbase/regionserver/wal/TestHBaseOnEC.java | 67 ++++++++++++++++++++++ 4 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHBaseOnEC.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java index bdf148e..7b9995c 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTe import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.util.Progressable; import org.apache.yetus.audience.InterfaceAudience; /** @@ -887,4 +888,16 @@ public abstract class CommonFSUtils { } } + public static FSDataOutputStream createHelper (FileSystem fs, Path f, boolean overwrite, int bufferSize, + short replication, long blockSize, Progressable progress) { + FSDataOutputStream output; + if (!(fs instanceof DistributedFileSystem)) { + output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, + null); + } else { + output = fs.create(path, overwritable, bufferSize).replicate().replication(replication, blockSize).build(); + } + return output; + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java index 6a7e4fa..d295555 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java @@ -66,7 +66,7 @@ public final class AsyncFSOutputHelper { int bufferSize = fs.getConf().getInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT); if (createParent) { - fsOut = fs.create(f, overwrite, bufferSize, replication, blockSize, null); + fsOut = CommonFSUtils.createHelper(fs, f, overwrite, bufferSize, replication, blockSize, null); } else { fsOut = fs.createNonRecursive(f, overwrite, bufferSize, replication, blockSize, null); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java index d1e72f7..de4e86c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java @@ -89,8 +89,12 @@ public class ProtobufLogWriter extends AbstractProtobufLogWriter @Override protected void initOutput(FileSystem fs, Path path, boolean overwritable, int bufferSize, short replication, long blockSize) throws IOException, StreamLacksCapabilityException { - this.output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, - null); + if (!(fs instanceof DistributedFileSystem)) { + this.output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, + null); + } else { + this.output = CommonFSUtils.createHelper(fs, f, overwrite, bufferSize, replication, blockSize, null); + } // TODO Be sure to add a check for hsync if this branch includes HBASE-19024 if (!(CommonFSUtils.hasCapability(output, "hflush"))) { throw new StreamLacksCapabilityException("hflush"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHBaseOnEC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHBaseOnEC.java new file mode 100644 index 0000000..ebce27c --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHBaseOnEC.java @@ -0,0 +1,67 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +// @Category LargeTest +public class TestHBaseOnEC { + static HBaseTestingUtility util = new HBaseTestingUtility(); + @BeforeClass + public void setup() { + Configuration conf = util.getConfiguration(); + util.startMiniCluster(); + MiniDFSCluster cluster = util.getDFSCluster(); + DFSTestUtil.enableAllECPolicies(cluster.getFileSystem()); + } + + @After + public void tearDown() { + util.shutdownMiniCluster(); + } + + @Test + public void testBasic() throws IOException { + String value = "value"; + + Table t = util.createTable(TableName.valueOf("TestHBaseOnEC"), "f"); + Put p = new Put("row".getBytes(), "columnFamily".getBytes(), + "qualifier".getBytes(), value.getBytes()); + t.put(p); + + byte[] retValue = t.get(new Get("row".getBytes())).getValue(); + assertEquals(value, Bytes.toString(retValue)); + } +} -- 2.7.4 (Apple Git-66)