diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 144f526..47b4977 100644
--- a/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -220,6 +220,11 @@ public final class HConstants {
// be the first to be reassigned if the server(s) they are being served by
// should go down.
+ /** version for metaupdated column where HTD is attached to HRI (pre-0.92)*/
+ public static final byte [] HTD_IN_HRI_VERSION = Bytes.toBytes((short)0);
+
+ /** version for metaupdated column where HTD is detached from HRI */
+ public static final byte [] NO_HTD_IN_HRI_VERSION = Bytes.toBytes((short)1);
//
// New stuff. Making a slow transition.
@@ -255,6 +260,17 @@ public final class HConstants {
/** The upper-half split region column qualifier */
public static final byte [] SPLITB_QUALIFIER = Bytes.toBytes("splitB");
+ /** The meta table version column qualifier */
+ public static final byte [] META_VERSION_QUALIFIER = Bytes.toBytes("v");
+
+ /**
+ * The version before this is the meta that had HTableDescriptor serialized
+ * into the HRegionInfo -- pre-hbase 0.92. There was no META_VERSION in this
+ * case. The presence of a version and its value being zero indicates we're
+ * on the new 0.92 HRegionInfo format.
+ */
+ public static final short META_VERSION = 0;
+
// Other constants
/**
diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
index 6387df9..050151b 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.util.Writables;
/**
* Writes region and assignment information to .META..
+ * TODO: Put MetaReader and MetaEditor together; doesn't make sense having
+ * them distinct.
*/
public class MetaEditor {
// TODO: Strip CatalogTracker from this class. Its all over and in the end
@@ -52,6 +54,12 @@ public class MetaEditor {
return put;
}
+ public static Put setMetaVersion(final Put p) {
+ p.add(HConstants.CATALOG_FAMILY, Bytes.toBytes(HConstants.META_VERSION),
+ Bytes.toBytes(HConstants.META_VERSION));
+ return p;
+ }
+
/**
* Put the passed p to the .META. table.
* @param ct CatalogTracker on whose back we will ride the edit.
diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java
index 0c1d221..8161d4b 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.migration.HRegionInfo090x;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
/**
@@ -43,10 +42,6 @@ import org.apache.hadoop.hbase.util.Writables;
public class MetaMigrationRemovingHTD {
private static final Log LOG = LogFactory.getLog(MetaMigrationRemovingHTD.class);
- /** The metaupdated column qualifier */
- public static final byte [] META_MIGRATION_QUALIFIER =
- Bytes.toBytes("metamigrated");
-
/**
* Update legacy META rows, removing HTD from HRI.
* @param masterServices
@@ -56,23 +51,10 @@ public class MetaMigrationRemovingHTD {
public static List updateMetaWithNewRegionInfo(
final MasterServices masterServices)
throws IOException {
- final List htds = new ArrayList();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r);
- if (hrfm == null) return true;
- htds.add(hrfm.getTableDesc());
- masterServices.getMasterFileSystem()
- .createTableDescriptor(hrfm.getTableDesc());
- updateHRI(masterServices.getCatalogTracker(), false, hrfm);
- return true;
- }
- };
+ MigratingVisitor v = new MigratingVisitor(masterServices, false);
MetaReader.fullScan(masterServices.getCatalogTracker(), v);
- MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker(), true);
- return htds;
+ updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker());
+ return v.htds;
}
/**
@@ -84,22 +66,34 @@ public class MetaMigrationRemovingHTD {
public static List updateRootWithNewRegionInfo(
final MasterServices masterServices)
throws IOException {
- final List htds = new ArrayList();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r);
- if (hrfm == null) return true;
- htds.add(hrfm.getTableDesc());
- masterServices.getMasterFileSystem().createTableDescriptor(
- hrfm.getTableDesc());
- updateHRI(masterServices.getCatalogTracker(), true, hrfm);
- return true;
- }
- };
+ MigratingVisitor v = new MigratingVisitor(masterServices, true);
MetaReader.fullScan(masterServices.getCatalogTracker(), v, null, true);
- return htds;
+ return v.htds;
+ }
+
+ /**
+ * Meta visitor that migrates the info:regioninfo as it visits.
+ */
+ static class MigratingVisitor implements Visitor {
+ private final MasterServices services;
+ // True if -ROOT- table and false if .META. table.
+ private final boolean meta;
+ final List htds = new ArrayList();
+ MigratingVisitor(final MasterServices services, final boolean meta) {
+ this.services = services;
+ this.meta = meta;
+ }
+
+ @Override
+ public boolean visit(Result r) throws IOException {
+ if (r == null || r.isEmpty()) return true;
+ HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
+ if (hrfm == null) return true;
+ htds.add(hrfm.getTableDesc());
+ this.services.getMasterFileSystem().createTableDescriptor(hrfm.getTableDesc());
+ updateHRI(this.services.getCatalogTracker(), this.meta, hrfm);
+ return true;
+ }
}
/**
@@ -115,18 +109,15 @@ public class MetaMigrationRemovingHTD {
}
/**
- * Update the metamigrated flag in -ROOT-.
+ * Update the version flag in -ROOT-.
* @param catalogTracker
- * @param metaUpdated
* @throws IOException
*/
- public static void updateRootWithMetaMigrationStatus(
- CatalogTracker catalogTracker, boolean metaUpdated)
+ public static void updateRootWithMetaMigrationStatus(final CatalogTracker catalogTracker)
throws IOException {
- Put p = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName());
- MetaMigrationRemovingHTD.addMetaUpdateStatus(p, metaUpdated);
- MetaEditor.putToRootTable(catalogTracker, p);
- LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated);
+ Put p = new Put(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
+ MetaEditor.putToRootTable(catalogTracker, MetaEditor.setMetaVersion(p));
+ LOG.info("Updated -ROOT- meta version=" + HConstants.META_VERSION);
}
static void updateHRI(final CatalogTracker ct, final boolean rootTable,
@@ -141,21 +132,20 @@ public class MetaMigrationRemovingHTD {
} else {
MetaEditor.putToMetaTable(ct, p);
}
- LOG.info("Updated region " + regionInfo + " to " +
- (rootTable? "-ROOT-": ".META."));
+ LOG.info("Migrated " + regionInfo + " in " + (rootTable? "-ROOT-": ".META."));
}
/**
* @deprecated Going away in 0.94; used for migrating to 0.92 only.
*/
- public static HRegionInfo090x getHRegionInfoForMigration(
- Result data) throws IOException {
+ public static HRegionInfo090x getHRegionInfoForMigration(Result data)
+ throws IOException {
HRegionInfo090x info = null;
byte [] bytes =
data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
if (bytes == null) return null;
try {
- info = Writables.getHRegionInfoForMigration(bytes);
+ info = getHRegionInfoForMigration(bytes);
} catch(IOException ioe) {
if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) {
return null;
@@ -187,13 +177,6 @@ public class MetaMigrationRemovingHTD {
return regions;
}
- static Put addMetaUpdateStatus(final Put p, final boolean metaUpdated) {
- p.add(HConstants.CATALOG_FAMILY,
- MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER,
- Bytes.toBytes(metaUpdated));
- return p;
- }
-
/**
* @return True if the meta table has been migrated.
* @throws IOException
@@ -201,22 +184,18 @@ public class MetaMigrationRemovingHTD {
// Public because used in tests
public static boolean isMetaHRIUpdated(final MasterServices services)
throws IOException {
- boolean metaUpdated = false;
List results =
MetaReader.fullScanOfRoot(services.getCatalogTracker());
if (results == null || results.isEmpty()) {
- LOG.info("metaUpdated = NULL.");
- return metaUpdated;
+ LOG.info("Not migrated");
+ return false;
}
// Presume only the one result.
Result r = results.get(0);
- byte [] metaMigrated = r.getValue(HConstants.CATALOG_FAMILY,
- MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER);
- if (metaMigrated != null && metaMigrated.length > 0) {
- metaUpdated = Bytes.toBoolean(metaMigrated);
- }
- LOG.info("Meta updated status = " + metaUpdated);
- return metaUpdated;
+ short version = MetaReader.getMetaVersion(r);
+ boolean migrated = version >= HConstants.META_VERSION;
+ LOG.info("Meta version=" + version + "; migrated=" + migrated);
+ return migrated;
}
/**
@@ -239,4 +218,15 @@ public class MetaMigrationRemovingHTD {
"Master startup aborted.");
}
}
+
+ /**
+ * Get HREgionInfoForMigration serialized from bytes.
+ * @param bytes serialized bytes
+ * @return HRegionInfoForMigration
+ * @throws IOException
+ */
+ public static HRegionInfo090x getHRegionInfoForMigration(final byte [] bytes)
+ throws IOException {
+ return (HRegionInfo090x)Writables.getWritable(bytes, new HRegionInfo090x());
+ }
}
\ No newline at end of file
diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
index f7f4d60..b9b8f3f 100644
--- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
+++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -63,6 +64,16 @@ public class MetaReader {
}
/**
+ * @param r Result to look at
+ * @return Current meta table version or -1 if no version found.
+ */
+ public static short getMetaVersion(final Result r) {
+ byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.META_VERSION_QUALIFIER);
+ return value == null || value.length <= 0? -1: Bytes.toShort(value);
+ }
+
+ /**
* @param regionName
* @return True if regionName is from .META. table.
*/
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index fcd071a..739d856 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3055,16 +3055,14 @@ public class HRegion implements HeapSize { // , Writable{
byte[] row = r.getRegionName();
Integer lid = meta.obtainRowLock(row);
try {
- final List edits = new ArrayList(1);
+ final long now = EnvironmentEdgeManager.currentTimeMillis();
+ final List edits = new ArrayList(2);
edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER,
- EnvironmentEdgeManager.currentTimeMillis(),
- Writables.getBytes(r.getRegionInfo())));
+ HConstants.REGIONINFO_QUALIFIER, now,
+ Writables.getBytes(r.getRegionInfo())));
edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
- org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER,
- EnvironmentEdgeManager.currentTimeMillis(),
- Bytes.toBytes(true)));
-
+ HConstants.META_VERSION_QUALIFIER, now,
+ Bytes.toBytes(HConstants.META_VERSION)));
meta.put(HConstants.CATALOG_FAMILY, edits);
} finally {
meta.releaseRowLock(lid);
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Writables.java b/src/main/java/org/apache/hadoop/hbase/util/Writables.java
index f595af5..1719e84 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Writables.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Writables.java
@@ -216,16 +216,4 @@ public class Writables {
}
return tgt;
}
-
- /**
- * Get HREgionInfoForMigration serialized from bytes.
- * @param bytes serialized bytes
- * @return HRegionInfoForMigration
- * @throws IOException
- */
- public static HRegionInfo090x getHRegionInfoForMigration(final byte [] bytes)
- throws IOException {
- return (HRegionInfo090x)getWritable(bytes, new HRegionInfo090x());
- }
-
}
\ No newline at end of file
diff --git a/src/test/data/hbase-4388-root.dir.tgz b/src/test/data/hbase-4388-root.dir.tgz
new file mode 100644
index 0000000..90c1033
Binary files /dev/null and b/src/test/data/hbase-4388-root.dir.tgz differ
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 6af3d35..dd60cc6 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -59,8 +59,8 @@ import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.ReadWriteConsistencyControl;
import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Threads;
@@ -72,13 +72,10 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.apache.zookeeper.ZooKeeper;
/**
* Facility for testing HBase. Replacement for
@@ -999,9 +996,18 @@ public class HBaseTestingUtility {
return count;
}
+ /**
+ * @param c
+ * @param htd
+ * @param family
+ * @param numRegions
+ * @return
+ * @throws IOException
+ * @deprecated Just for testing migration of meta from 0.90 to 0.92... will be
+ * removed thereafter
+ */
public int createMultiRegionsWithLegacyHRI(final Configuration c,
- final HTableDescriptor htd,
- final byte [] family, int numRegions)
+ final HTableDescriptor htd, final byte [] family, int numRegions)
throws IOException {
if (numRegions < 3) throw new IOException("Must create at least 3 regions");
byte [] startKey = Bytes.toBytes("aaaaa");
@@ -1015,9 +1021,18 @@ public class HBaseTestingUtility {
return createMultiRegionsWithLegacyHRI(c, htd, family, regionStartKeys);
}
+ /**
+ * @param c
+ * @param htd
+ * @param columnFamily
+ * @param startKeys
+ * @return
+ * @throws IOException
+ * @deprecated Just for testing migration of meta from 0.90 to 0.92... will be
+ * removed thereafter
+ */
public int createMultiRegionsWithLegacyHRI(final Configuration c,
- final HTableDescriptor htd,
- final byte[] columnFamily, byte [][] startKeys)
+ final HTableDescriptor htd, final byte[] columnFamily, byte [][] startKeys)
throws IOException {
Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
deleted file mode 100644
index 6cbf88c..0000000
--- a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * Copyright 2010 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client;
-
-import junit.framework.AssertionFailedError;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
-import org.apache.hadoop.hbase.migration.HRegionInfo090x;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
-
-import org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD;
-import org.apache.hadoop.hbase.catalog.MetaReader;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.apache.hadoop.hbase.util.Writables;
-
-import java.util.List;
-
-public class TestMetaMigration {
- final Log LOG = LogFactory.getLog(getClass());
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private static MiniHBaseCluster miniHBaseCluster = null;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- miniHBaseCluster = TEST_UTIL.startMiniCluster(1);
- }
-
- /**
- * @throws java.lang.Exception
- */
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
- }
-
- @Test
- public void testHRegionInfoForMigration() throws Exception {
- LOG.info("Starting testHRegionInfoForMigration");
- HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
- htd.addFamily(new HColumnDescriptor("family"));
- HRegionInfo090x hrim = new HRegionInfo090x(htd, HConstants.EMPTY_START_ROW,
- HConstants.EMPTY_END_ROW);
- LOG.info("INFO 1 = " + hrim);
- byte[] bytes = Writables.getBytes(hrim);
- LOG.info(" BYtes.toString = " + Bytes.toString(bytes));
- LOG.info(" HTD bytes = " + Bytes.toString(Writables.getBytes(hrim.getTableDesc())));
- HRegionInfo090x info = Writables.getHRegionInfoForMigration(bytes);
- LOG.info("info = " + info);
- LOG.info("END testHRegionInfoForMigration");
-
- }
-
- @Test
- public void testMetaUpdatedFlagInROOT() throws Exception {
- LOG.info("Starting testMetaUpdatedFlagInROOT");
- boolean metaUpdated =
- MetaMigrationRemovingHTD.isMetaHRIUpdated(miniHBaseCluster.getMaster());
- assertEquals(true, metaUpdated);
- LOG.info("END testMetaUpdatedFlagInROOT");
- }
-
- @Test
- public void testMetaMigration() throws Exception {
- LOG.info("Starting testMetaWithLegacyHRI");
- final byte[] FAMILY = Bytes.toBytes("family");
- HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
- HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
- htd.addFamily(hcd);
- Configuration conf = TEST_UTIL.getConfiguration();
- TEST_UTIL.createMultiRegionsWithLegacyHRI(conf, htd, FAMILY,
- new byte[][]{
- HConstants.EMPTY_START_ROW,
- Bytes.toBytes("region_a"),
- Bytes.toBytes("region_b")});
- CatalogTracker ct = miniHBaseCluster.getMaster().getCatalogTracker();
- // just for this test set it to false.
- MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct, false);
- MetaReader.fullScanMetaAndPrint(ct);
- LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI");
-
- List htds = MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
- TEST_UTIL.getHBaseCluster().getMaster());
- MetaReader.fullScanMetaAndPrint(ct);
- assertEquals(3, htds.size());
- // Assert that the flag in ROOT is updated to reflect the correct status
- boolean metaUpdated =
- MetaMigrationRemovingHTD.isMetaHRIUpdated(miniHBaseCluster.getMaster());
- assertEquals(true, metaUpdated);
- LOG.info("END testMetaWithLegacyHRI");
-
- }
-
- /**
- * This test assumes a master crash/failure during the meta migration process
- * and attempts to continue the meta migration process when a new master takes over.
- * When a master dies during the meta migration we will have some rows of
- * META.CatalogFamily updated with new HRI, (i.e HRI with out HTD) and some
- * still hanging with legacy HRI. (i.e HRI with HTD). When the backup master/ or
- * fresh start of master attempts the migration it will encouter some rows of META
- * already updated with new HRI and some still legacy. This test will simulate this
- * scenario and validates that the migration process can safely skip the updated
- * rows and migrate any pending rows at startup.
- * @throws Exception
- */
- @Test
- public void testMasterCrashDuringMetaMigration() throws Exception {
- LOG.info("Starting testMasterCrashDuringMetaMigration");
- final byte[] FAMILY = Bytes.toBytes("family");
- HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration");
- HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
- htd.addFamily(hcd);
- Configuration conf = TEST_UTIL.getConfiguration();
- // Create 10 New regions.
- TEST_UTIL.createMultiRegionsWithNewHRI(conf, htd, FAMILY, 10);
- // Create 10 Legacy regions.
- TEST_UTIL.createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, 10);
- CatalogTracker ct = miniHBaseCluster.getMaster().getCatalogTracker();
- // just for this test set it to false.
- MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct, false);
- //MetaReader.fullScanMetaAndPrint(ct);
- LOG.info("MEta Print completed.testUpdatesOnMetaWithLegacyHRI");
-
- List htds = MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
- TEST_UTIL.getHBaseCluster().getMaster());
- assertEquals(10, htds.size());
- // Assert that the flag in ROOT is updated to reflect the correct status
- boolean metaUpdated =
- MetaMigrationRemovingHTD.isMetaHRIUpdated(miniHBaseCluster.getMaster());
- assertEquals(true, metaUpdated);
- LOG.info("END testMetaWithLegacyHRI");
-
- }
-
- public static void assertEquals(int expected,
- int actual) {
- if (expected != actual) {
- throw new AssertionFailedError("expected:<" +
- expected + "> but was:<" +
- actual + ">");
- }
- }
-
- public static void assertEquals(boolean expected,
- boolean actual) {
- if (expected != actual) {
- throw new AssertionFailedError("expected:<" +
- expected + "> but was:<" +
- actual + ">");
- }
- }
-
-
-
-}
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
new file mode 100644
index 0000000..74b9938
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
@@ -0,0 +1,206 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import junit.framework.AssertionFailedError;
+
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.migration.HRegionInfo090x;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+
+import org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.apache.hadoop.hbase.util.Writables;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Test migration that removes HTableDescriptor from HRegionInfo moving the
+ * meta version from no version to {@link MetaReader#META_VERSION}.
+ */
+public class TestMetaMigrationRemovingHTD {
+ final Log LOG = LogFactory.getLog(getClass());
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ // Start up our mini cluster on top of an 0.90 root.dir. See if we can
+ // migrate from 0.90.
+ TEST_UTIL.startMiniZKCluster();
+ TEST_UTIL.startMiniDFSCluster(1);
+ Path testdir = HBaseTestingUtility.getTestDir("TestMetaMigrationRemovingHTD");
+ // Find the src data under src/test/data
+ final String datafile = "hbase-4388-root.dir";
+ String srcTarFile =
+ System.getProperty("test.cache.data", "build/test/cache") + datafile + ".tgz";
+ File tgtUntarFile = new File(testdir.toString(), datafile);
+ File untardir = new File(testdir.to)
+ FileUtil.unTar(inFile, untarDir)(tarFile, testdir);
+
+ if (TEST_UTIL)
+ File dfsDir = new File(dataDir, "dfs");
+if ( dfsDir.exists() && !FileUtil.fullyDelete(dfsDir) ) {
+throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
+}
+FileUtil.unTar(new File(tarFile), new File(dataDir));
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testSerializingHRegionInfo090x() throws Exception {
+ LOG.info("Starting testHRegionInfoForMigration");
+ HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
+ htd.addFamily(new HColumnDescriptor("family"));
+ HRegionInfo090x hrim = new HRegionInfo090x(htd, HConstants.EMPTY_START_ROW,
+ HConstants.EMPTY_END_ROW);
+ byte[] bytes = Writables.getBytes(hrim);
+ MetaMigrationRemovingHTD.getHRegionInfoForMigration(bytes);
+ }
+
+ @Test
+ public void testMetaUpdatedFlagInROOT() throws Exception {
+ boolean metaUpdated = MetaMigrationRemovingHTD.
+ isMetaHRIUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster());
+ assertEquals(true, metaUpdated);
+ }
+
+ @Test
+ public void testMetaMigration() throws Exception {
+ LOG.info("Starting testMetaWithLegacyHRI");
+ final byte [] FAMILY = Bytes.toBytes("family");
+ HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
+ HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+ htd.addFamily(hcd);
+ Configuration conf = TEST_UTIL.getConfiguration();
+ TEST_UTIL.createMultiRegionsWithLegacyHRI(conf, htd, FAMILY,
+ new byte[][]{
+ HConstants.EMPTY_START_ROW,
+ Bytes.toBytes("region_a"),
+ Bytes.toBytes("region_b")});
+ CatalogTracker ct =
+ TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
+ // Erase the current version of root meta for this test.
+ undoVersionInMeta();
+ MetaReader.fullScanMetaAndPrint(ct);
+ LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI");
+
+ List htds =
+ MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
+ TEST_UTIL.getHBaseCluster().getMaster());
+ MetaReader.fullScanMetaAndPrint(ct);
+ assertEquals(3, htds.size());
+ // Assert that the flag in ROOT is updated to reflect the correct status
+ boolean metaUpdated =
+ MetaMigrationRemovingHTD.isMetaHRIUpdated(
+ TEST_UTIL.getMiniHBaseCluster().getMaster());
+ assertEquals(true, metaUpdated);
+ }
+
+ /**
+ * This test assumes a master crash/failure during the meta migration process
+ * and attempts to continue the meta migration process when a new master takes over.
+ * When a master dies during the meta migration we will have some rows of
+ * META.CatalogFamily updated with new HRI, (i.e HRI with out HTD) and some
+ * still hanging with legacy HRI. (i.e HRI with HTD). When the backup master/ or
+ * fresh start of master attempts the migration it will encouter some rows of META
+ * already updated with new HRI and some still legacy. This test will simulate this
+ * scenario and validates that the migration process can safely skip the updated
+ * rows and migrate any pending rows at startup.
+ * @throws Exception
+ */
+ @Test
+ public void testMasterCrashDuringMetaMigration() throws Exception {
+ final byte[] FAMILY = Bytes.toBytes("family");
+ HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration");
+ HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+ htd.addFamily(hcd);
+ Configuration conf = TEST_UTIL.getConfiguration();
+ // Create 10 New regions.
+ TEST_UTIL.createMultiRegionsWithNewHRI(conf, htd, FAMILY, 10);
+ // Create 10 Legacy regions.
+ TEST_UTIL.createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, 10);
+ CatalogTracker ct =
+ TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
+ Delete d = new Delete(HRegionInfo.ROOT_REGIONINFO.getRegionName());
+ // Erase the current version of root meta for this test.
+ undoVersionInMeta();
+ MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct);
+ //MetaReader.fullScanMetaAndPrint(ct);
+ LOG.info("MEta Print completed.testUpdatesOnMetaWithLegacyHRI");
+
+ List htds = MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
+ TEST_UTIL.getHBaseCluster().getMaster());
+ assertEquals(10, htds.size());
+ // Assert that the flag in ROOT is updated to reflect the correct status
+ boolean metaUpdated = MetaMigrationRemovingHTD.
+ isMetaHRIUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster());
+ assertEquals(true, metaUpdated);
+ LOG.info("END testMetaWithLegacyHRI");
+ }
+
+ private void undoVersionInMeta() throws IOException {
+ Delete d = new Delete(HRegionInfo.ROOT_REGIONINFO.getRegionName());
+ // Erase the current version of root meta for this test.
+ d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER);
+ HTable rootTable =
+ new HTable(TEST_UTIL.getConfiguration(), HConstants.ROOT_TABLE_NAME);
+ try {
+ rootTable.delete(d);
+ } finally {
+ rootTable.close();
+ }
+ }
+
+ public static void assertEquals(int expected, int actual) {
+ if (expected != actual) {
+ throw new AssertionFailedError("expected:<" +
+ expected + "> but was:<" +
+ actual + ">");
+ }
+ }
+
+ public static void assertEquals(boolean expected, boolean actual) {
+ if (expected != actual) {
+ throw new AssertionFailedError("expected:<" +
+ expected + "> but was:<" +
+ actual + ">");
+ }
+ }
+}
\ No newline at end of file