Index: bin/hbase =================================================================== --- bin/hbase (revision 1513783) +++ bin/hbase (working copy) @@ -86,6 +86,7 @@ echo " thrift run the HBase Thrift server" echo " thrift2 run the HBase Thrift2 server" echo " clean run the HBase clean up script" + echo " migrate run the HBase migrate script" echo "" echo "PACKAGE MANAGEMENT" echo " classpath dump hbase CLASSPATH" @@ -317,6 +318,13 @@ fi "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@ exit $? + elif [ "$COMMAND" = "migrate" ] ; then + "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} --migrate + exitCode=$? + if [ "$exitCode" = "0" ]; then + "$bin"/hbase org.apache.hadoop.hbase.util.HFileV1Detector $@ + fi + exit $? elif [ "$COMMAND" = "classpath" ] ; then echo $CLASSPATH Index: bin/hbase-cleanup.sh =================================================================== --- bin/hbase-cleanup.sh (revision 1513783) +++ bin/hbase-cleanup.sh (working copy) @@ -31,7 +31,7 @@ # HADOOP_SSH_OPTS Options passed to ssh when running remote commands. # -usage="Usage: hbase-cleanup.sh (zk|hdfs|all)" +usage="Usage: hbase-cleanup.sh (--cleanAll|--cleanZk|--cleanHdfs|--migrate)" bin=`dirname "$0"` bin=`cd "$bin">/dev/null; pwd` @@ -40,7 +40,7 @@ . "$bin"/hbase-config.sh case $1 in - --cleanZk|--cleanHdfs|--cleanAll) + --cleanZk|--cleanHdfs|--cleanAll|--migrate) matches="yes" ;; *) ;; esac @@ -109,6 +109,13 @@ execute_zk_command "rmr ${zparent}"; execute_hdfs_command "-rmr ${hrootdir}" ;; + --migrate) + "$bin"/hbase org.apache.hadoop.hbase.migration.NamespaceUpgrade --upgrade + "$bin"/hbase org.apache.hadoop.hbase.util.ZKDataMigrator + hbaseidFile="hbase.id" + execute_hdfs_command "-rmr ${hrootdir}/$hbaseidFile" + ;; + *) ;; esac Index: hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java (revision 1513783) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java (working copy) @@ -427,7 +427,7 @@ * for use as content of a this.peersZNode; i.e. the content of PEER_ID znode under * /hbase/replication/peers/PEER_ID */ - private static byte[] toByteArray(final String clusterKey) { + public static byte[] toByteArray(final String clusterKey) { byte[] bytes = ZooKeeperProtos.ReplicationPeer.newBuilder().setClusterkey(clusterKey).build() .toByteArray(); Index: hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java (revision 1513783) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java (working copy) @@ -54,7 +54,7 @@ // Public for testing public static final byte[] ENABLED_ZNODE_BYTES = toByteArray(ZooKeeperProtos.ReplicationState.State.ENABLED); - protected static final byte[] DISABLED_ZNODE_BYTES = + public static final byte[] DISABLED_ZNODE_BYTES = toByteArray(ZooKeeperProtos.ReplicationState.State.DISABLED); public ReplicationStateZKBase(ZooKeeperWatcher zookeeper, Configuration conf, Index: hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java (revision 1513783) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java (working copy) @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; @@ -51,6 +52,7 @@ import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; import com.google.common.collect.Lists; import com.google.common.primitives.Ints; @@ -486,4 +488,9 @@ public Configuration getConf() { return conf; } + + public static void main(String[] args) throws Exception { + System.exit(ToolRunner.run(HBaseConfiguration.create(), + new NamespaceUpgrade(), args)); + } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java (revision 1513783) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java (working copy) @@ -129,6 +129,7 @@ @Override public int run(String args[]) throws IOException, ParseException { fs = FileSystem.get(getConf()); + FSUtils.setFsDefault(getConf(), new Path(fs.getUri())); numOfThreads = DEFAULT_NUM_OF_THREADS; dirToProcess = FSUtils.getRootDir(getConf()); if (!parseOption(args)) { Index: hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java (revision 0) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/ZKDataMigrator.java (working copy) @@ -0,0 +1,285 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; +import org.apache.hadoop.hbase.replication.ReplicationPeersZKImpl; +import org.apache.hadoop.hbase.replication.ReplicationStateZKBase; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NoNodeException; + +/** + * Tool to migrate zookeeper data of older hbase versions(<0.95.0) to PB. + */ +public class ZKDataMigrator extends Configured implements Tool { + + private static final Log LOG = LogFactory.getLog(ZKDataMigrator.class); + + @Override + public int run(String[] as) throws Exception { + Configuration conf = getConf(); + ZooKeeperWatcher zkw = null; + try { + zkw = new ZooKeeperWatcher(getConf(), "Migrate ZK data to PB.", + new ZKDataMigratorAbortable()); + FileSystem fs = FileSystem.get(getConf()); + FSUtils.setFsDefault(conf, new Path(fs.getUri())); + if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) { + System.out + .println("No hbase related data available in zookeeper. returning.."); + return 0; + } + List children = ZKUtil.listChildrenNoWatch(zkw, zkw.baseZNode); + if (children == null) { + System.out.println("No child nodes to mirgrate. returning.."); + return 0; + } + String childPath = null; + for (String child : children) { + childPath = ZKUtil.joinZNode(zkw.baseZNode, child); + if (child.equals(conf.get("zookeeper.znode.rootserver", + "root-region-server"))) { + // -ROOT- region no longer present from 0.95.0, so we can remove this + // znode + ZKUtil.deleteNodeRecursively(zkw, childPath); + // TODO delete root table path from file system. + } else if (child.equals(conf.get("zookeeper.znode.rs", "rs"))) { + // Since there is no live region server instance during migration, we + // can remove this znode as well. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.draining.rs", + "draining"))) { + // If we want to migrate to 0.95.0 from older versions we need to stop + // the existing cluster. So there wont be any draining servers so we + // can + // remove it. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.master", "master"))) { + // Since there is no live master instance during migration, we can + // remove this znode as well. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.backup.masters", + "backup-masters"))) { + // Since there is no live backup master instances during migration, we + // can remove this znode as well. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.state", "shutdown"))) { + // shutdown node is not present from 0.95.0 onwards. Its renamed to + // "running". We can delete it. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.unassigned", + "unassigned"))) { + // Any way during clean cluster startup we will remove all unassigned + // region nodes. we can delete all children nodes as well. This znode + // is + // renamed to "regions-in-transition" from 0.95.0 onwards. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.tableEnableDisable", + "table")) + || child.equals(conf.get( + "zookeeper.znode.masterTableEnableDisable", "table"))) { + checkAndMigrateTableStatesToPB(zkw); + } else if (child.equals(conf.get( + "zookeeper.znode.masterTableEnableDisable92", "table92"))) { + // This is replica of table states from tableZnode so we can remove + // this. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf + .get("zookeeper.znode.splitlog", "splitlog"))) { + // This znode no longer available from 0.95.0 onwards, we can remove + // it. + ZKUtil.deleteNodeRecursively(zkw, childPath); + } else if (child.equals(conf.get("zookeeper.znode.replication", + "replication"))) { + checkAndMigrateReplicationNodesToPB(zkw); + } + } + } catch (Exception e) { + System.err.println(e); + return -1; + } finally { + if (zkw != null) { + zkw.close(); + } + } + return 0; + } + + private void checkAndMigrateTableStatesToPB(ZooKeeperWatcher zkw) + throws KeeperException { + List tables = ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode); + if (tables == null) { + System.out + .println("No table present to migrate table state to PB. returning.."); + } + for (String table : tables) { + String znode = ZKUtil.joinZNode(zkw.tableZNode, table); + // Delete -ROOT- table state znode since its no longer present in 0.95.0 + // onwards. + if (table.equals("-ROOT-") || table.equals(".META.")) { + ZKUtil.deleteNode(zkw, znode); + continue; + } + byte[] data = ZKUtil.getData(zkw, znode); + if (ProtobufUtil.isPBMagicPrefix(data)) + continue; + ZooKeeperProtos.Table.Builder builder = ZooKeeperProtos.Table + .newBuilder(); + builder + .setState(ZooKeeperProtos.Table.State.valueOf(Bytes.toString(data))); + data = ProtobufUtil.prependPBMagic(builder.build().toByteArray()); + ZKUtil.setData(zkw, znode, data); + } + } + + private void checkAndMigrateReplicationNodesToPB(ZooKeeperWatcher zkw) + throws KeeperException { + String replicationZnodeName = getConf().get("zookeeper.znode.replication", + "replication"); + String replicationPath = ZKUtil.joinZNode(zkw.baseZNode, + replicationZnodeName); + List replicationZnodes = ZKUtil.listChildrenNoWatch(zkw, + replicationPath); + if (replicationZnodes == null) { + System.out + .println("No replication related znodes present to migrate. returning.."); + } + for (String child : replicationZnodes) { + String znode = ZKUtil.joinZNode(replicationPath, child); + if (child.equals(getConf().get("zookeeper.znode.replication.peers", + "peers"))) { + List peers = ZKUtil.listChildrenNoWatch(zkw, znode); + if (peers == null || peers.isEmpty()) { + System.out.println("No peers present to migrate. returning.."); + continue; + } + checkAndMigratePeerZnodesToPB(zkw, znode, peers); + } else if (child.equals(getConf().get( + "zookeeper.znode.replication.state", "state"))) { + // This is no longer used in >=0.95.x + ZKUtil.deleteNodeRecursively(zkw, znode); + } else if (child.equals(getConf().get("zookeeper.znode.replication.rs", + "rs"))) { + List rsList = ZKUtil.listChildrenNoWatch(zkw, znode); + if (rsList == null || rsList.isEmpty()) continue; + for (String rs : rsList) { + checkAndMigrateQueuesToPB(zkw, znode, rs); + } + } + } + } + + private void checkAndMigrateQueuesToPB(ZooKeeperWatcher zkw, String znode, String rs) + throws KeeperException, NoNodeException { + String rsPath = ZKUtil.joinZNode(znode, rs); + List peers = ZKUtil.listChildrenNoWatch(zkw, rsPath); + if (peers == null || peers.isEmpty()) + return; + String peerPath = null; + for (String peer : peers) { + peerPath = ZKUtil.joinZNode(rsPath, peer); + List files = ZKUtil.listChildrenNoWatch(zkw, peerPath); + if (files == null || files.isEmpty()) continue; + String filePath = null; + for (String file : files) { + filePath = ZKUtil.joinZNode(peerPath, file); + byte[] data = ZKUtil.getData(zkw, filePath); + if(data == null || Bytes.equals(data, HConstants.EMPTY_BYTE_ARRAY)) continue; + if (ProtobufUtil.isPBMagicPrefix(data)) continue; + ZKUtil.setData(zkw, filePath, ZKUtil.positionToByteArray(Long + .parseLong(Bytes.toString(data)))); + } + } + } + + private void checkAndMigratePeerZnodesToPB(ZooKeeperWatcher zkw, String znode, + List peers) throws KeeperException, NoNodeException { + for (String peer : peers) { + String peerZnode = ZKUtil.joinZNode(znode, peer); + byte[] data = ZKUtil.getData(zkw, peerZnode); + if (!ProtobufUtil.isPBMagicPrefix(data)) { + migrateClusterKeyToPB(zkw, peerZnode, data); + } + String peerStatePath = ZKUtil.joinZNode( + peerZnode, + getConf().get("zookeeper.znode.replication.peers.state", + "peer-state")); + if (ZKUtil.checkExists(zkw, peerStatePath) != -1) { + data = ZKUtil.getData(zkw, peerStatePath); + if (ProtobufUtil.isPBMagicPrefix(data)) continue; + migratePeerStateToPB(zkw, data, peerStatePath); + } + } + } + + private void migrateClusterKeyToPB(ZooKeeperWatcher zkw, String peerZnode, + byte[] data) throws KeeperException, NoNodeException { + ZKUtil.setData(zkw, peerZnode, + ReplicationPeersZKImpl.toByteArray(Bytes.toString(data))); + } + + private void migratePeerStateToPB(ZooKeeperWatcher zkw, byte[] data, + String peerStatePath) throws KeeperException, NoNodeException { + String state = Bytes.toString(data); + if (ZooKeeperProtos.ReplicationState.State.ENABLED.name().equals( + state)) { + ZKUtil.setData(zkw, peerStatePath, + ReplicationStateZKBase.ENABLED_ZNODE_BYTES); + } else if (ZooKeeperProtos.ReplicationState.State.DISABLED.name() + .equals(state)) { + ZKUtil.setData(zkw, peerStatePath, + ReplicationStateZKBase.DISABLED_ZNODE_BYTES); + } + } + + public static void main(String args[]) throws Exception { + System.exit(ToolRunner.run(HBaseConfiguration.create(), + new ZKDataMigrator(), args)); + } + + static class ZKDataMigratorAbortable implements Abortable { + private boolean aborted = false; + @Override + public void abort(String why, Throwable e) { + System.err.println("Got aborted with reason: " + why + ", and error: " + e); + this.aborted = true; + } + + @Override + public boolean isAborted() { + return this.aborted; + } + } + +} Index: hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java (revision 1513783) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java (working copy) @@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Durability; -import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.util.ToolRunner; @@ -179,15 +178,6 @@ } @Test - public void testMetaUpdatedFlagInROOT() throws Exception { - HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); - boolean metaUpdated = MetaMigrationConvertingToPB. - isMetaTableUpdated(master.getCatalogTracker()); - assertEquals(true, metaUpdated); - verifyMetaRowsAreUpdated(master.getCatalogTracker()); - } - - @Test public void testMetaMigration() throws Exception { LOG.info("Starting testMetaMigration"); final byte [] FAMILY = Bytes.toBytes("family");