Index: bin/hbase =================================================================== --- bin/hbase (revision 1516633) +++ bin/hbase (working copy) @@ -77,6 +77,7 @@ echo " hlog write-ahead-log analyzer" echo " hfile store file analyzer" echo " zkcli run the ZooKeeper shell" + echo " upgrade upgrade hbase" echo "" echo "PROCESS MANAGEMENT" echo " master run an HBase HMaster node" @@ -271,6 +272,8 @@ CLASS='org.apache.hadoop.hbase.io.hfile.HFile' elif [ "$COMMAND" = "zkcli" ] ; then CLASS="org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer" +elif [ "$COMMAND" = "upgrade" ] ; then + CLASS="org.apache.hadoop.hbase.migration.UpgradeTo96" elif [ "$COMMAND" = "master" ] ; then CLASS='org.apache.hadoop.hbase.master.HMaster' if [ "$1" != "stop" ] ; then Index: hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java (revision 1516633) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java (working copy) @@ -54,7 +54,7 @@ // Public for testing public static final byte[] ENABLED_ZNODE_BYTES = toByteArray(ZooKeeperProtos.ReplicationState.State.ENABLED); - protected static final byte[] DISABLED_ZNODE_BYTES = + public static final byte[] DISABLED_ZNODE_BYTES = toByteArray(ZooKeeperProtos.ReplicationState.State.DISABLED); public ReplicationStateZKBase(ZooKeeperWatcher zookeeper, Configuration conf, Index: hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java (revision 1516633) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/migration/NamespaceUpgrade.java (working copy) @@ -105,6 +105,7 @@ public void init() throws IOException { this.rootDir = FSUtils.getRootDir(conf); + FSUtils.setFsDefault(getConf(), rootDir); this.fs = FileSystem.get(conf); Path tmpDataDir = new Path(rootDir, TMP_DATA_DIR); sysNsDir = new Path(tmpDataDir, NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java (revision 1516633) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java (working copy) @@ -46,7 +46,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.io.HFileLink; -import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.util.Tool; @@ -69,9 +68,27 @@ private static final Log LOG = LogFactory.getLog(HFileV1Detector.class); private static final int DEFAULT_NUM_OF_THREADS = 10; private int numOfThreads; - private Path dirToProcess; + /** + * directory to start the processing. + */ + private Path targetDirPath; + /** + * executor for processing regions. + */ + private ExecutorService exec; + + /** + * Keeps record of processed tables. + */ + private final Set processedTables = new HashSet(); + /** + * set of corrupted HFiles (with undetermined major version) + */ private final Set corruptedHFiles = Collections .newSetFromMap(new ConcurrentHashMap()); + /** + * set of HfileV1; + */ private final Set hFileV1Set = Collections .newSetFromMap(new ConcurrentHashMap()); @@ -107,75 +124,101 @@ } if (cmd.hasOption("p")) { - dirToProcess = new Path(cmd.getOptionValue("p")); + this.targetDirPath = new Path(FSUtils.getRootDir(getConf()), cmd.getOptionValue("p")); } try { if (cmd.hasOption("n")) { int n = Integer.parseInt(cmd.getOptionValue("n")); if (n < 0 || n > 100) { - System.out.println("Please use a positive number <= 100 for number of threads." + LOG.warn("Please use a positive number <= 100 for number of threads." + " Continuing with default value " + DEFAULT_NUM_OF_THREADS); return true; } - numOfThreads = n; + this.numOfThreads = n; } } catch (NumberFormatException nfe) { - System.err.println("Please select a valid number for threads"); + LOG.error("Please select a valid number for threads"); return false; } return true; } + /** + * Checks for HFileV1. + * @return 0 when no HFileV1 is present. + * 1 when a HFileV1 is present or, when there is a file with corrupt major version + * (neither V1 nor V2). + * -1 in case of any error/exception + */ @Override public int run(String args[]) throws IOException, ParseException { + FSUtils.setFsDefault(getConf(), new Path(FSUtils.getRootDir(getConf()).toUri())); fs = FileSystem.get(getConf()); numOfThreads = DEFAULT_NUM_OF_THREADS; - dirToProcess = FSUtils.getRootDir(getConf()); + targetDirPath = FSUtils.getRootDir(getConf()); if (!parseOption(args)) { - System.exit(1); + System.exit(-1); } - ExecutorService exec = Executors.newFixedThreadPool(numOfThreads); - Set regionsWithHFileV1; + this.exec = Executors.newFixedThreadPool(numOfThreads); try { - regionsWithHFileV1 = checkForV1Files(dirToProcess, exec); - printHRegionsWithHFileV1(regionsWithHFileV1); - printAllHFileV1(); - printCorruptedHFiles(); - if (hFileV1Set.isEmpty() && corruptedHFiles.isEmpty()) { - // all clear. - System.out.println("No HFile V1 Found"); - } + return processResult(checkForV1Files(targetDirPath)); } catch (Exception e) { - System.err.println(e); - return 1; + LOG.error(e); } finally { exec.shutdown(); fs.close(); } - return 0; + return -1; } + private int processResult(Set regionsWithHFileV1) { + LOG.info("Result: \n"); + printSet(processedTables, "Tables Processed: "); + + int count = hFileV1Set.size(); + LOG.info("Count of HFileV1: " + count); + if (count > 0) printSet(hFileV1Set, "HFileV1:"); + + count = corruptedHFiles.size(); + LOG.info("Count of corrupted files: " + count); + if (count > 0) printSet(corruptedHFiles, "Corrupted Files: "); + + count = regionsWithHFileV1.size(); + LOG.info("Count of Regions with HFileV1: " + count); + if (count > 0) printSet(regionsWithHFileV1, "Regions to Major Compact: "); + + return (hFileV1Set.isEmpty() && corruptedHFiles.isEmpty()) ? 0 : 1; + } + + private void printSet(Set result, String msg) { + LOG.info(msg); + for (Path p : result) { + LOG.info(p); + } + } + /** * Takes a directory path, and lists out any HFileV1, if present. * @param targetDir directory to start looking for HFilev1. - * @param exec * @return set of Regions that have HFileV1 * @throws IOException */ - private Set checkForV1Files(Path targetDir, final ExecutorService exec) throws IOException { - if (isTableDir(fs, targetDir)) { - return processTable(targetDir, exec); - } - // user has passed a hbase installation directory. + private Set checkForV1Files(Path targetDir) throws IOException { + LOG.info("Target dir is: " + targetDir); if (!fs.exists(targetDir)) { throw new IOException("The given path does not exist: " + targetDir); } + if (isTableDir(fs, targetDir)) { + processedTables.add(targetDir); + return processTable(targetDir); + } Set regionsWithHFileV1 = new HashSet(); FileStatus[] fsStats = fs.listStatus(targetDir); for (FileStatus fsStat : fsStats) { - if (isTableDir(fs, fsStat.getPath())) { + if (isTableDir(fs, fsStat.getPath()) && !isRootTable(fsStat.getPath())) { + processedTables.add(fsStat.getPath()); // look for regions and find out any v1 file. - regionsWithHFileV1.addAll(processTable(fsStat.getPath(), exec)); + regionsWithHFileV1.addAll(processTable(fsStat.getPath())); } else { LOG.info("Ignoring path: " + fsStat.getPath()); } @@ -184,15 +227,24 @@ } /** - * Find out the regions in the table which has an HFile v1 in it. + * Ignore ROOT table as it doesn't exist in 0.96. + * @param path + * @return + */ + private boolean isRootTable(Path path) { + if (path != null && path.toString().endsWith("-ROOT-")) return true; + return false; + } + + /** + * Find out regions in the table which have HFileV1. * @param tableDir - * @param exec * @return the set of regions containing HFile v1. * @throws IOException */ - private Set processTable(Path tableDir, final ExecutorService exec) throws IOException { + private Set processTable(Path tableDir) throws IOException { // list out the regions and then process each file in it. - LOG.info("processing table: " + tableDir); + LOG.debug("processing table: " + tableDir); List> regionLevelResults = new ArrayList>(); Set regionsWithHFileV1 = new HashSet(); @@ -200,7 +252,7 @@ for (FileStatus fsStat : fsStats) { // process each region if (isRegionDir(fs, fsStat.getPath())) { - regionLevelResults.add(processRegion(fsStat.getPath(), exec)); + regionLevelResults.add(processRegion(fsStat.getPath())); } } for (Future f : regionLevelResults) { @@ -209,9 +261,9 @@ regionsWithHFileV1.add(f.get()); } } catch (InterruptedException e) { - System.err.println(e); + LOG.error(e); } catch (ExecutionException e) { - System.err.println(e); // might be a bad hfile. We print it at the end. + LOG.error(e); // might be a bad hfile. We print it at the end. } } return regionsWithHFileV1; @@ -221,11 +273,10 @@ * Each region is processed by a separate handler. If a HRegion has a hfileV1, its path is * returned as the future result, otherwise, a null value is returned. * @param regionDir Region to process. - * @param exec * @return corresponding Future object. */ - private Future processRegion(final Path regionDir, final ExecutorService exec) { - LOG.info("processing region: " + regionDir); + private Future processRegion(final Path regionDir) { + LOG.debug("processing region: " + regionDir); Callable regionCallable = new Callable() { @Override public Path call() throws Exception { @@ -249,15 +300,17 @@ fsdis = fs.open(storeFilePath); lenToRead = storeFile.getLen(); } - FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis, lenToRead); - int version = trailer.getMajorVersion(); - if (version == 1) { + int majorVersion = computeMajorVersion(fsdis, lenToRead); + if (majorVersion == 1) { hFileV1Set.add(storeFilePath); // return this region path, as it needs to be compacted. return regionDir; } + if (majorVersion > 2 || majorVersion < 1) throw new IllegalArgumentException( + "Incorrect major version: " + majorVersion); } catch (Exception iae) { corruptedHFiles.add(storeFilePath); + LOG.error("Got exception while reading trailer for file: "+ storeFilePath, iae); } finally { if (fsdis != null) fsdis.close(); } @@ -265,13 +318,30 @@ } return null; } + + private int computeMajorVersion(FSDataInputStream istream, long fileSize) + throws IOException { + //read up the last int of the file. Major version is in the last 3 bytes. + long seekPoint = fileSize - Bytes.SIZEOF_INT; + if (seekPoint < 0) + throw new IllegalArgumentException("File too small, no major version found"); + + // Read the version from the last int of the file. + istream.seek(seekPoint); + int version = istream.readInt(); + // Extract and return the major version + return version & 0x00ffffff; + } }; Future f = exec.submit(regionCallable); return f; } private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException { - return FSTableDescriptors.getTableInfoPath(fs, path) != null; + // check for old format, of having /table/.tableinfo; .META. doesn't has .tableinfo, + // include it. + return (FSTableDescriptors.getTableInfoPath(fs, path) != null || FSTableDescriptors + .getCurrentTableInfoStatus(fs, path, false) != null) || path.toString().endsWith(".META."); } private static boolean isRegionDir(final FileSystem fs, final Path path) throws IOException { @@ -280,43 +350,6 @@ } - private void printHRegionsWithHFileV1(Set regionsHavingHFileV1) { - if (!regionsHavingHFileV1.isEmpty()) { - System.out.println(); - System.out.println("Following regions has HFileV1 and needs to be Major Compacted:"); - System.out.println(); - for (Path r : regionsHavingHFileV1) { - System.out.println(r); - } - System.out.println(); - } - } - - private void printAllHFileV1() { - if (!hFileV1Set.isEmpty()) { - System.out.println(); - System.out.println("Following HFileV1 are found:"); - System.out.println(); - for (Path r : hFileV1Set) { - System.out.println(r); - } - System.out.println(); - } - - } - - private void printCorruptedHFiles() { - if (!corruptedHFiles.isEmpty()) { - System.out.println(); - System.out.println("Following HFiles are corrupted as their version is unknown:"); - System.out.println(); - for (Path r : corruptedHFiles) { - System.out.println(r); - } - System.out.println(); - } - } - public static void main(String args[]) throws Exception { System.exit(ToolRunner.run(HBaseConfiguration.create(), new HFileV1Detector(), args)); } Index: hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java (revision 1516633) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java (working copy) @@ -150,7 +150,7 @@ } } - private static File untar(final File testdir) throws IOException { + static File untar(final File testdir) throws IOException { // Find the src data under src/test/data final String datafile = "TestNamespaceUpgrade"; File srcTarFile = new File(