diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java index 09cf8b9..2fbd6eb 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java @@ -72,8 +72,18 @@ public final class TableName implements Comparable { public static final TableName NAMESPACE_TABLE_NAME = valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "namespace"); - private static final String OLD_META_STR = ".META."; - private static final String OLD_ROOT_STR = "-ROOT-"; + public static final String OLD_META_STR = ".META."; + public static final String OLD_ROOT_STR = "-ROOT-"; + + /** + * TableName for old -ROOT- table. It is used to read/process old WALs which have + * ROOT edits. + */ + public static final TableName OLD_ROOT_TABLE_NAME = getADummyTableName(OLD_ROOT_STR); + /** + * TableName for old .META. table. Used in testing. + */ + public static final TableName OLD_META_TABLE_NAME = getADummyTableName(OLD_META_STR); private byte[] name; private String nameAsString; @@ -231,6 +241,18 @@ public final class TableName implements Comparable { return ret; } + /** + * It is used to create table names for old META, and ROOT table. + * @return a dummy TableName instance (with no validation) for the passed qualifier + */ + private static TableName getADummyTableName(String qualifier) { + TableName ret = new TableName(); + ret.namespaceAsString = NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR; + ret.qualifierAsString = qualifier; + ret.nameAsString = createFullyQualified(ret.namespaceAsString, ret.qualifierAsString); + ret.name = Bytes.toBytes(qualifier); + return ret; + } public static TableName valueOf(String namespaceAsString, String qualifierAsString) { TableName ret = new TableName(); if(namespaceAsString == null || namespaceAsString.length() < 1) { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java index 481ecc6..d0434e9 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java @@ -31,6 +31,7 @@ import java.util.UUID; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; @@ -322,6 +323,7 @@ public class HLogKey implements WritableComparable { // @see Bytes#readByteArray(DataInput) this.scopes = null; // writable HLogKey does not contain scopes int len = WritableUtils.readVInt(in); + byte[] tablenameBytes = null; if (len < 0) { // what we just read was the version version = Version.fromCode(len); @@ -334,12 +336,10 @@ public class HLogKey implements WritableComparable { if (compressionContext == null || !version.atLeast(Version.COMPRESSED)) { this.encodedRegionName = new byte[len]; in.readFully(this.encodedRegionName); - byte[] tablenameBytes = Bytes.readByteArray(in); - this.tablename = TableName.valueOf(tablenameBytes); + tablenameBytes = Bytes.readByteArray(in); } else { this.encodedRegionName = Compressor.readCompressed(in, compressionContext.regionDict); - byte[] tablenameBytes = Compressor.readCompressed(in, compressionContext.tableDict); - this.tablename = TableName.valueOf(tablenameBytes); + tablenameBytes = Compressor.readCompressed(in, compressionContext.tableDict); } this.logSeqNum = in.readLong(); @@ -357,6 +357,19 @@ public class HLogKey implements WritableComparable { // Means it's a very old key, just continue } } + try { + this.tablename = TableName.valueOf(tablenameBytes); + } catch (IllegalArgumentException iae) { + if (Bytes.toString(tablenameBytes).equals(TableName.OLD_META_STR)) { + // It is a pre-namespace meta table edit, continue with new format. + LOG.info("Got an old META edit, continuing with new format ", iae); + this.tablename = TableName.META_TABLE_NAME; + this.encodedRegionName = HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); + } else if (Bytes.toString(tablenameBytes).equals(TableName.OLD_ROOT_STR)) { + this.tablename = TableName.OLD_ROOT_TABLE_NAME; + throw iae; + } else throw iae; + } } public WALKey.Builder getBuilder( diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java index 2b9337b..1a0fdde 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java @@ -21,16 +21,20 @@ package org.apache.hadoop.hbase.regionserver.wal; import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALTrailer; import org.apache.hadoop.hbase.util.FSUtils; @InterfaceAudience.Private public abstract class ReaderBase implements HLog.Reader { + private static final Log LOG = LogFactory.getLog(ReaderBase.class); protected Configuration conf; protected FileSystem fs; protected Path path; @@ -95,7 +99,18 @@ public abstract class ReaderBase implements HLog.Reader { e.setCompressionContext(compressionContext); } - boolean hasEntry = readNext(e); + boolean hasEntry = false; + try { + hasEntry = readNext(e); + } catch (IllegalArgumentException iae) { + TableName tableName = e.getKey().getTablename(); + if (tableName != null && tableName.equals(TableName.OLD_ROOT_TABLE_NAME)) { + // It is old ROOT table edit, ignore it + LOG.info("Got an old ROOT edit, ignoring ", iae); + return next(e); + } + else throw iae; + } edit++; if (compressionContext != null && emptyCompressionContext) { emptyCompressionContext = false; @@ -103,7 +118,6 @@ public abstract class ReaderBase implements HLog.Reader { return hasEntry ? e : null; } - @Override public void seek(long pos) throws IOException { if (compressionContext != null && emptyCompressionContext) { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadOldRootAndMetaEdits.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadOldRootAndMetaEdits.java new file mode 100644 index 0000000..217ed29 --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadOldRootAndMetaEdits.java @@ -0,0 +1,160 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests to read old ROOT, Meta edits. + */ +@Category(MediumTests.class) + +public class TestReadOldRootAndMetaEdits { + + private final static Log LOG = LogFactory.getLog(TestReadOldRootAndMetaEdits.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static FileSystem fs; + private static Path dir; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setClass("hbase.regionserver.hlog.writer.impl", + SequenceFileLogWriter.class, HLog.Writer.class); + TEST_UTIL.startMiniDFSCluster(3); + fs = TEST_UTIL.getTestFileSystem(); + dir = new Path(TEST_UTIL.createRootDir(), "testReadOldRootAndMetaEdits"); + fs.mkdirs(dir); + + } + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Inserts three waledits in the wal file, and reads them back. The first edit is of a regular + * table, second waledit is for the ROOT table (it will be ignored while reading), + * and last waledit is for the META table, which will be linked to the new system:meta table. + * @throws IOException + */ + @Test + public void testReadOldRootAndMetaEdits() throws IOException { + LOG.debug("testReadOldRootAndMetaEdits"); + Configuration conf = HBaseConfiguration.create(); + conf.setClass("hbase.regionserver.hlog.writer.impl", SequenceFileLogWriter.class, + HLog.Writer.class); + // kv list to be used for all WALEdits. + byte[] row = Bytes.toBytes("row"); + KeyValue kv = new KeyValue(row, row, row, row); + List kvs = new ArrayList(); + kvs.add(kv); + + HLog.Writer writer = null; + HLog.Reader reader = null; + // a regular table + TableName t = TableName.valueOf("t"); + HRegionInfo tRegionInfo = null; + int logCount = 0; + long timestamp = System.currentTimeMillis(); + Path path = new Path(dir, "t"); + try { + tRegionInfo = new HRegionInfo(t, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + HLog.Entry tEntry = createAEntry(new HLogKey(tRegionInfo.getEncodedNameAsBytes(), t, + ++logCount, timestamp, HConstants.DEFAULT_CLUSTER_ID), kvs); + + // create a old root edit (-ROOT-). + HLog.Entry rootEntry = createAEntry(new HLogKey(Bytes.toBytes(TableName.OLD_ROOT_STR), + TableName.OLD_ROOT_TABLE_NAME, ++logCount, timestamp, + HConstants.DEFAULT_CLUSTER_ID), kvs); + + // create a old meta edit (.META.). + HLog.Entry oldMetaEntry = createAEntry(new HLogKey(Bytes.toBytes(TableName.OLD_META_STR), + TableName.OLD_META_TABLE_NAME, ++logCount, timestamp, + HConstants.DEFAULT_CLUSTER_ID), kvs); + + // write above entries + writer = HLogFactory.createWriter(fs, path, conf); + writer.append(tEntry); + writer.append(rootEntry); + writer.append(oldMetaEntry); + + // sync/close the writer + writer.sync(); + writer.close(); + + // read the log and see things are okay. + reader = HLogFactory.createReader(fs, path, conf); + HLog.Entry entry = reader.next(); + assertNotNull(entry); + assertTrue(entry.getKey().getTablename().equals(t)); + assertEquals(Bytes.toString(entry.getKey().getEncodedRegionName()), + Bytes.toString(tRegionInfo.getEncodedNameAsBytes())); + + // read the ROOT waledit, but that will be ignored, and META waledit will be read instead. + entry = reader.next(); + assertEquals(entry.getKey().getTablename(), TableName.META_TABLE_NAME); + // should reach end of log + assertNull(reader.next()); + } finally { + if (writer != null) { + writer.close(); + } + if (reader != null) { + reader.close(); + } + } +} + /** + * Creates a WALEdit for the passed KeyValues and returns a HLog.Entry instance composed of + * the WALEdit and passed HLogKey. + * @return HLog.Entry instance for the passed HLogKey and KeyValues + */ + private HLog.Entry createAEntry(HLogKey hlogKey, List kvs) { + WALEdit edit = new WALEdit(); + for (KeyValue kv : kvs ) + edit.add(kv); + return new HLog.Entry(hlogKey, edit); + } + +}