From e4dd226d21671878cb655ef7982be0e3c023f94a Mon Sep 17 00:00:00 2001 From: Guanghao Zhang Date: Wed, 24 Oct 2018 14:27:31 +0800 Subject: [PATCH] HBASE-21365 Throw exception when user put data with skip wal to a table which may be replicated --- .../InvalidMutationDurabilityException.java | 44 ++++++ .../apache/hadoop/hbase/regionserver/HRegion.java | 49 ++++--- .../TestInvalidMutationDurabilityException.java | 156 +++++++++++++++++++++ 3 files changed, 226 insertions(+), 23 deletions(-) create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/InvalidMutationDurabilityException.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestInvalidMutationDurabilityException.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/InvalidMutationDurabilityException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/InvalidMutationDurabilityException.java new file mode 100644 index 0000000..d694f37 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/InvalidMutationDurabilityException.java @@ -0,0 +1,44 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Thrown if Mutation's {@link Durability} is skip wal but table need replication. + */ +@InterfaceAudience.Public +public class InvalidMutationDurabilityException extends DoNotRetryIOException { + + /** + * default constructor + */ + public InvalidMutationDurabilityException() { + super(); + } + + /** + * @param message exception message + */ + public InvalidMutationDurabilityException(String message) { + super(message); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index fc33512..c10a824 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -2949,7 +2949,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi if(family == null) { throw new NoSuchColumnFamilyException("Empty family is invalid"); } - checkFamily(family); + checkFamily(family, delete.getDurability()); } } } @@ -3561,7 +3561,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi @Override public void checkAndPreparePut(Put p) throws IOException { - region.checkFamilies(p.getFamilyCellMap().keySet()); + region.checkFamilies(p.getFamilyCellMap().keySet(), p.getDurability()); } @Override @@ -4454,14 +4454,30 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi store.add(cell, memstoreAccounting); } - /** - * Check the collection of families for validity. - * @param families - * @throws NoSuchColumnFamilyException - */ - public void checkFamilies(Collection families) throws NoSuchColumnFamilyException { + private void checkFamilies(Collection families, Durability durability) + throws NoSuchColumnFamilyException, InvalidMutationDurabilityException { for (byte[] family : families) { - checkFamily(family); + checkFamily(family, durability); + } + } + + private void checkFamily(final byte[] family, Durability durability) + throws NoSuchColumnFamilyException, InvalidMutationDurabilityException { + checkFamily(family); + if (durability.equals(Durability.SKIP_WAL) + && htableDescriptor.getColumnFamily(family).getScope() + != HConstants.REPLICATION_SCOPE_LOCAL) { + throw new InvalidMutationDurabilityException( + "Mutation's durability is SKIP_WAL but table's column family " + Bytes.toString(family) + + " need replication"); + } + } + + void checkFamily(final byte[] family) throws NoSuchColumnFamilyException { + if (!this.htableDescriptor.hasColumnFamily(family)) { + throw new NoSuchColumnFamilyException( + "Column family " + Bytes.toString(family) + " does not exist in region " + this + + " in table " + this.htableDescriptor); } } @@ -7762,7 +7778,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi checkReadOnly(); checkResources(); checkRow(mutation.getRow(), op.toString()); - checkFamilies(mutation.getFamilyCellMap().keySet()); + checkFamilies(mutation.getFamilyCellMap().keySet(), mutation.getDurability()); this.writeRequestsCount.increment(); WriteEntry writeEntry = null; startRegionOperation(op); @@ -8112,19 +8128,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi return cells; } - // - // New HBASE-880 Helpers - // - - void checkFamily(final byte [] family) - throws NoSuchColumnFamilyException { - if (!this.htableDescriptor.hasColumnFamily(family)) { - throw new NoSuchColumnFamilyException("Column family " + - Bytes.toString(family) + " does not exist in region " + this - + " in table " + this.htableDescriptor); - } - } - public static final long FIXED_OVERHEAD = ClassSize.align( ClassSize.OBJECT + ClassSize.ARRAY + diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestInvalidMutationDurabilityException.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestInvalidMutationDurabilityException.java new file mode 100644 index 0000000..e22f6dc --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestInvalidMutationDurabilityException.java @@ -0,0 +1,156 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.InvalidMutationDurabilityException; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MediumTests.class, ClientTests.class }) +public class TestInvalidMutationDurabilityException { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestInvalidMutationDurabilityException.class); + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static TableName TABLE_NOT_REPLICATE = TableName.valueOf("TableNotReplicate"); + + private static TableName TABLE_NEED_REPLICATE = TableName.valueOf("TableNeedReplicate"); + + private static byte[] CF = Bytes.toBytes("cf"); + + private static byte[] CQ = Bytes.toBytes("cq"); + + private static Table tableNotReplicate; + + private static Table tableNeedReplicate; + + @BeforeClass + public static void setUp() throws Exception { + UTIL.startMiniCluster(); + UTIL.getAdmin().createTable(TableDescriptorBuilder.newBuilder(TABLE_NOT_REPLICATE) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(CF).build()).build()); + UTIL.getAdmin().createTable(TableDescriptorBuilder.newBuilder(TABLE_NEED_REPLICATE) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(CF) + .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build()); + tableNotReplicate = UTIL.getConnection().getTable(TABLE_NOT_REPLICATE); + tableNeedReplicate = UTIL.getConnection().getTable(TABLE_NEED_REPLICATE); + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.getAdmin().disableTable(TABLE_NOT_REPLICATE); + UTIL.getAdmin().disableTable(TABLE_NEED_REPLICATE); + UTIL.getAdmin().deleteTable(TABLE_NOT_REPLICATE); + UTIL.getAdmin().deleteTable(TABLE_NEED_REPLICATE); + UTIL.shutdownMiniCluster(); + } + + private Put newPutWithSkipWAL() { + Put put = new Put(Bytes.toBytes("row")); + put.addColumn(CF, CQ, Bytes.toBytes("value")); + put.setDurability(Durability.SKIP_WAL); + return put; + } + + @Test + public void testPutToTableNotReplicate() throws Exception { + tableNotReplicate.put(newPutWithSkipWAL()); + } + + @Test(expected = InvalidMutationDurabilityException.class) + public void testPutToTableNeedReplicate() throws Exception { + tableNeedReplicate.put(newPutWithSkipWAL()); + } + + private Delete newDeleteWithSkipWAL() { + Delete delete = new Delete(Bytes.toBytes("row")); + delete.addColumn(CF, CQ); + delete.setDurability(Durability.SKIP_WAL); + return delete; + } + + @Test + public void testDeleteToTableNotReplicate() throws Exception { + tableNotReplicate.delete(newDeleteWithSkipWAL()); + } + + @Test(expected = InvalidMutationDurabilityException.class) + public void testDeleteToTableNeedReplicate() throws Exception { + tableNeedReplicate.delete(newDeleteWithSkipWAL()); + } + + private Append newAppendWithSkipWAL() { + Append append = new Append(Bytes.toBytes("row")); + append.addColumn(CF, CQ, Bytes.toBytes("value")); + append.setDurability(Durability.SKIP_WAL); + return append; + } + + @Test + public void testAppendToTableNotReplicate() throws Exception { + tableNotReplicate.append(newAppendWithSkipWAL()); + } + + @Test(expected = InvalidMutationDurabilityException.class) + public void testAppendToTableNeedReplicate() throws Exception { + tableNeedReplicate.append(newAppendWithSkipWAL()); + } + + private Increment newIncrementWithSkipWAL() { + Increment increment = new Increment(Bytes.toBytes("row")); + increment.addColumn(CF, CQ, 1); + increment.setDurability(Durability.SKIP_WAL); + return increment; + } + + @Test + public void testIncrementToTableNotReplicate() throws Exception { + tableNotReplicate.increment(newIncrementWithSkipWAL()); + } + + @Test(expected = InvalidMutationDurabilityException.class) + public void testIncrementToTableNeedReplicate() throws Exception { + tableNeedReplicate.increment(newIncrementWithSkipWAL()); + } + + @Test + public void testCheckWithMutateToTableNotReplicate() throws Exception { + tableNotReplicate.checkAndMutate(Bytes.toBytes("row"), CF).qualifier(CQ).ifNotExists() + .thenPut(newPutWithSkipWAL()); + } + + @Test(expected = InvalidMutationDurabilityException.class) + public void testCheckWithMutateToTableNeedReplicate() throws Exception { + tableNeedReplicate.checkAndMutate(Bytes.toBytes("row"), CF).qualifier(CQ).ifNotExists() + .thenPut(newPutWithSkipWAL()); + } +} -- 2.7.4