Index: bin/hbase =================================================================== --- bin/hbase (revision 1502983) +++ bin/hbase (working copy) @@ -86,6 +86,7 @@ echo " thrift run the HBase Thrift server" echo " thrift2 run the HBase Thrift2 server" echo " clean run the HBase clean up script" + echo " migrate run the HBase migrate script" echo "" echo "PACKAGE MANAGEMENT" echo " classpath dump hbase CLASSPATH" @@ -317,6 +318,13 @@ fi "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@ exit $? + elif [ "$COMMAND" = "migrate" ] ; then + "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} --migrate + exitCode=$? + if [ "$exitCode" = "0" ]; then + "$bin"/hbase org.apache.hadoop.hbase.util.HFileV1Detector $@ + fi + exit $? elif [ "$COMMAND" = "classpath" ] ; then echo $CLASSPATH Index: bin/hbase-cleanup.sh =================================================================== --- bin/hbase-cleanup.sh (revision 1502983) +++ bin/hbase-cleanup.sh (working copy) @@ -31,7 +31,7 @@ # HADOOP_SSH_OPTS Options passed to ssh when running remote commands. # -usage="Usage: hbase-cleanup.sh (zk|hdfs|all)" +usage="Usage: hbase-cleanup.sh (--cleanAll|--cleanZk|--cleanHdfs|--migrate)" bin=`dirname "$0"` bin=`cd "$bin">/dev/null; pwd` @@ -40,7 +40,7 @@ . "$bin"/hbase-config.sh case $1 in - --cleanZk|--cleanHdfs|--cleanAll) + --cleanZk|--cleanHdfs|--cleanAll|--migrate) matches="yes" ;; *) ;; esac @@ -109,6 +109,12 @@ execute_zk_command "rmr ${zparent}"; execute_hdfs_command "-rmr ${hrootdir}" ;; + --migrate) + "$bin"/hbase org.apache.hadoop.hbase.util.ZKDataMigrator + rootTable = "-ROOT-" + execute_hdfs_command "-rmr ${hrootdir}/$rootTable" + ;; + *) ;; esac Index: hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java (revision 0) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java (working copy) @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.zookeeper.KeeperException; + +/** + * Tool to migrate zookeeper data of older hbase versions(<0.95.0) to PB. + */ +public class ZKDataMigrator extends Configured implements Tool { + + private static final Log LOG = LogFactory.getLog(ZKDataMigrator.class); + + @Override + public int run(String[] as) throws Exception { + Configuration conf = getConf(); + ZooKeeperWatcher zkw = null; + try { + zkw = new ZooKeeperWatcher(getConf(), "Migrate ZK data to PB.", + new Abortable() { + @Override + public void abort(String why, Throwable e) {} + @Override + public boolean isAborted() { return false; } + }); + String rootDir = getConf().get("hbase.rootdir"); + String defaultFs = rootDir.substring(0, rootDir.lastIndexOf('/')); + conf.set("fs.default.name", conf.get("fs.default.name", defaultFs)); + conf.set("fs.defaultFS", conf.get("fs.defaultFS", defaultFs)); + if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) { + System.out + .println("No hbase related data available in zookeeper. returning.."); + return 0; + } + List children = ZKUtil.listChildrenNoWatch(zkw, zkw.baseZNode); + if (children == null) { + System.out.println("No child nodes to mirgrate. returning.."); + return 0; + } + for (String child : children) { + String childPath = ZKUtil.joinZNode(zkw.baseZNode, child); + if (child.equals(conf.get("zookeeper.znode.rootserver", + "root-region-server"))) { + // -ROOT- region no longer present from 0.95.0, so we can remove this + // znode after creating meta region server znode with -META- location + // from -ROOT- + // table(we will scan from filesystem). Otherwise we may assign -META- + // region before splitting its HLog. + createMetaRegionServerZnode(zkw); + ZKUtil.deleteNodeRecursively(zkw, childPath); + // TODO delete root table path from file system. + } else if (child.equals(conf.get("zookeeper.znode.rs", "rs"))) { + // Since there is no live region server instance during migration, we + // can remove this znode as well. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.draining.rs", + "draining"))) { + // If we want to migrate to 0.95.0 from older versions we need to stop + // the existing cluster. So there wont be any draining servers so we + // can + // remove it. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.master", "master"))) { + // Since there is no live master instance during migration, we can + // remove this znode as well. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.backup.masters", + "backup-masters"))) { + // Since there is no live backup master instances during migration, we + // can remove this znode as well. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.state", "shutdown"))) { + // shutdown node is not present from 0.95.0 onwards. Its renamed to + // "running". We can delete it. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.unassigned", + "unassigned"))) { + // Any way during clean cluster startup we will remove all unassigned + // region nodes. we can delete all children nodes as well. This znode + // is + // renamed to "regions-in-transition" from 0.95.0 onwards. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.tableEnableDisable", + "table")) + || child.equals(conf.get( + "zookeeper.znode.masterTableEnableDisable", "table"))) { + checkAndMigrateTableStatesToPB(zkw); + } else if (child.equals(conf.get( + "zookeeper.znode.masterTableEnableDisable92", "table92"))) { + // This is replica of table states from tableZnode so we can remove + // this. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf + .get("zookeeper.znode.clusterId", "hbaseid"))) { + // If we start cluster again id will be changed. We can remove this. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf + .get("zookeeper.znode.splitlog", "splitlog"))) { + // This znode no longer available from 0.95.0 onwards, we can remove + // it. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } + } + } catch (Exception e) { + System.err.println(e); + return -1; + } finally { + if (zkw != null) { + zkw.close(); + } + } + return 0; + } + + private void createMetaRegionServerZnode(ZooKeeperWatcher zkw) + throws IOException, KeeperException { + HRegion region = null; + RegionScanner scanner = null; + try { + region = HRegion.openHRegion(HRegionInfo.ROOT_REGIONINFO, + HTableDescriptor.ROOT_TABLEDESC, null, getConf()); + Scan s = new Scan(); + scanner = region.getScanner(s); + List results = new ArrayList(); + scanner.next(results); + String server = null; + long serverStartCode = -1; + for (KeyValue kv : results) { + if (Bytes.equals(kv.getQualifier(), HConstants.SERVER_QUALIFIER)) { + server = Bytes.toString(kv.getValue()); + } else if (Bytes.equals(kv.getQualifier(), + HConstants.STARTCODE_QUALIFIER)) { + serverStartCode = Bytes.toLong(kv.getValue()); + } + } + MetaRegionTracker.setMetaLocation(zkw, new ServerName(server, + serverStartCode)); + } finally { + if (region.getLog() != null) { + region.getLog().close(); + } + if (region != null) { + region.close(); + } + if (scanner != null) { + scanner.close(); + } + } + } + + private void checkAndMigrateTableStatesToPB(ZooKeeperWatcher zkw) + throws KeeperException { + List tables = ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode); + if (tables == null) { + System.out + .println("No table present to migrate table state to PB. returning.."); + } + for (String table : tables) { + String znode = ZKUtil.joinZNode(zkw.tableZNode, table); + // Delete -ROOT- table state znode since its no longer present in 0.95.0 + // onwards. + if (table.equals("-ROOT-")) { + ZKUtil.deleteNodeRecursively(zkw, znode); + continue; + } + byte[] data = ZKUtil.getData(zkw, znode); + if (ProtobufUtil.isPBMagicPrefix(data)) + continue; + ZooKeeperProtos.Table.Builder builder = ZooKeeperProtos.Table + .newBuilder(); + builder + .setState(ZooKeeperProtos.Table.State.valueOf(Bytes.toString(data))); + data = ProtobufUtil.prependPBMagic(builder.build().toByteArray()); + ZKUtil.setData(zkw, znode, data); + } + } + + public static void main(String args[]) throws Exception { + System.exit(ToolRunner.run(HBaseConfiguration.create(), + new ZKDataMigrator(), args)); + } + +} Index: hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java (revision 1502983) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java (working copy) @@ -170,15 +170,6 @@ } @Test - public void testMetaUpdatedFlagInROOT() throws Exception { - HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); - boolean metaUpdated = MetaMigrationConvertingToPB. - isMetaTableUpdated(master.getCatalogTracker()); - assertEquals(true, metaUpdated); - verifyMetaRowsAreUpdated(master.getCatalogTracker()); - } - - @Test public void testMetaMigration() throws Exception { LOG.info("Starting testMetaMigration"); final byte [] FAMILY = Bytes.toBytes("family");