From 6b5053304931092cb99d59adf4db763e14d90d33 Mon Sep 17 00:00:00 2001 From: Sakthi Date: Fri, 23 Feb 2018 12:09:19 -0800 Subject: [PATCH] HBASE-18864 Fixed NullPointerException thrown while adding rows to a table from peer cluster, with replication factor other than 0 or 1 --- .../java/org/apache/hadoop/hbase/wal/WALKey.java | 18 ++++++++-- .../hadoop/hbase/wal/TestDefaultWALProvider.java | 38 ++++++++++++++++++---- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java index 05acd7208818f9c1769e6d0b748a63a18944b936..7eab4b02354be3a8a33b1de1d7b38fc0593682b0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKey.java @@ -198,6 +198,9 @@ public class WALKey implements SequenceId, Comparable { private MultiVersionConcurrencyControl.WriteEntry writeEntry; public static final List EMPTY_UUIDS = Collections.unmodifiableList(new ArrayList()); + // If replication scope is invalid then log warning once. + private static boolean loggedReplicationScopeInvalid = false; + // visible for deprecated HLogKey @InterfaceAudience.Private protected CompressionContext compressionContext; @@ -615,8 +618,19 @@ public class WALKey implements SequenceId, Comparable { for (Map.Entry e : scopes.entrySet()) { ByteString family = (compressionContext == null) ? ByteStringer.wrap(e.getKey()) : compressor.compress(e.getKey(), compressionContext.familyDict); - builder.addScopes(FamilyScope.newBuilder() - .setFamily(family).setScopeType(ScopeType.valueOf(e.getValue()))); + ScopeType scopeType = ScopeType.valueOf(e.getValue()); + if (scopeType == null) { + // Any value other than '1' for the REPLICATION_SCOPE will be considered as + // REPLICATION_SCOPE_LOCAL (0) + if (!loggedReplicationScopeInvalid) { + // log this only once + LOG.warn("WALKey=" + this + ". Invalid value for REPLICATION_SCOPE=" + + e.getValue() + ". Using REPLICATION_SCOPE_LOCAL (0) by default."); + loggedReplicationScopeInvalid = true; + } + scopeType = ScopeType.REPLICATION_SCOPE_LOCAL; + } + builder.addScopes(FamilyScope.newBuilder().setFamily(family).setScopeType(scopeType)); } } return builder; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java index 44029094e58b29f42043508fccdf518e8f306e9e..d1187d03be35861221cea41d99d9652f42aa8d89 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java @@ -18,15 +18,11 @@ */ package org.apache.hadoop.hbase.wal; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - import java.io.IOException; import java.util.HashSet; import java.util.Random; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; @@ -41,11 +37,11 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.protobuf.generated.WALProtos; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.junit.After; @@ -60,6 +56,8 @@ import org.junit.rules.TestName; // imports for things that haven't moved from regionserver.wal yet. import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import static org.junit.Assert.*; + @Category(MediumTests.class) public class TestDefaultWALProvider { private static final Log LOG = LogFactory.getLog(TestDefaultWALProvider.class); @@ -118,6 +116,34 @@ public class TestDefaultWALProvider { return "TestDefaultWALProvider"; } + @Test + public void testInvalidWALKeyScope() throws IOException { + HTableDescriptor table = new HTableDescriptor(TableName.valueOf("t1")); + table.addFamily(new HColumnDescriptor("f1")); + + final Configuration localConf = new Configuration(conf); + localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName()); + + final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName()); + final WAL wal = wals.getWAL(UNSPECIFIED_REGION); + assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal)); + HRegionInfo hri = + new HRegionInfo(table.getTableName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + WALKey walKey = + getWalKey(hri.getEncodedNameAsBytes(), table.getTableName(), System.currentTimeMillis()); + TreeMap scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); + + // Set invalid value for replication scope + scopes.put(Bytes.toBytes("REPLICATION_SCOPE"), new Integer(5)); + walKey.setScopes(scopes); + WALProtos.WALKey.Builder walKeyBuilder = walKey.getBuilder(null); + assertEquals(1, walKeyBuilder.getScopesCount()); + + WALProtos.ScopeType scopeType = walKeyBuilder.getScopesList().get(0).getScopeType(); + assertNotNull(scopeType); + assertEquals(WALProtos.ScopeType.REPLICATION_SCOPE_LOCAL, scopeType); + } + @Test public void testGetServerNameFromWALDirectoryName() throws IOException { ServerName sn = ServerName.valueOf("hn", 450, 1398); -- 2.16.2