diff --git src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java index 5052878..4c31a1e 100644 --- src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java +++ src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.DeserializationException; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; @@ -344,9 +345,14 @@ public class MasterFileSystem { } // as above FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, - 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS, - HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS)); + 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS, + HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS)); } + } catch (DeserializationException de) { + LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de); + IOException ioe = new IOException(); + ioe.initCause(de); + throw ioe; } catch (IllegalArgumentException iae) { LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR + " " + rd.toString(), iae); diff --git src/main/java/org/apache/hadoop/hbase/util/FSUtils.java src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index 3d35d3e..1c66cf5 100644 --- src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -19,10 +19,12 @@ */ package org.apache.hadoop.hbase.util; +import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -43,17 +45,23 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.DeserializationException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.RemoteExceptionHandler; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.FSProtos; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; +import com.google.protobuf.InvalidProtocolBufferException; + /** * Utility methods for interacting with the underlying file system. */ @@ -252,26 +260,75 @@ public abstract class FSUtils { * @param rootdir root hbase directory * @return null if no version file exists, version string otherwise. * @throws IOException e + * @throws DeserializationException */ public static String getVersion(FileSystem fs, Path rootdir) - throws IOException { + throws IOException, DeserializationException { Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME); + FileStatus [] status = fs.listStatus(versionFile); + if (status == null || status.length == 0) return null; String version = null; - if (fs.exists(versionFile)) { - FSDataInputStream s = - fs.open(versionFile); - try { - version = DataInputStream.readUTF(s); - } catch (EOFException eof) { - LOG.warn("Version file was empty, odd, will try to set it."); - } finally { - s.close(); + byte [] content = new byte [(int)status[0].getLen()]; + FSDataInputStream s = fs.open(versionFile); + try { + IOUtils.readFully(s, content, 0, content.length); + if (ProtobufUtil.isPBMagicPrefix(content)) { + version = parseVersionFrom(content); + } else { + // Presume it pre-pb format. + InputStream is = new ByteArrayInputStream(content); + DataInputStream dis = new DataInputStream(is); + try { + version = dis.readUTF(); + } finally { + dis.close(); + } + // Update the format + LOG.info("Updating the hbase.version file format with version=" + version); + setVersion(fs, rootdir, version, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS); } + } catch (EOFException eof) { + LOG.warn("Version file was empty, odd, will try to set it."); + } finally { + s.close(); } return version; } /** + * Parse the content of the ${HBASE_ROOTDIR}/hbase.version file. + * @param bytes The byte content of the hbase.version file. + * @return The version found in the file as a String. + * @throws DeserializationException + */ + static String parseVersionFrom(final byte [] bytes) + throws DeserializationException { + ProtobufUtil.expectPBMagicPrefix(bytes); + int pblen = ProtobufUtil.lengthOfPBMagic(); + FSProtos.HBaseVersionFileContent.Builder builder = + FSProtos.HBaseVersionFileContent.newBuilder(); + FSProtos.HBaseVersionFileContent fileContent; + try { + fileContent = builder.mergeFrom(bytes, pblen, bytes.length - pblen).build(); + return fileContent.getVersion(); + } catch (InvalidProtocolBufferException e) { + // Convert + throw new DeserializationException(e); + } + } + + /** + * Create the content to write into the ${HBASE_ROOTDIR}/hbase.version file. + * @param version Version to persist + * @return Serialized protobuf with version content and a bit of pb magic for a prefix. + */ + static byte [] toVersionByteArray(final String version) { + FSProtos.HBaseVersionFileContent.Builder builder = + FSProtos.HBaseVersionFileContent.newBuilder(); + return ProtobufUtil.prependPBMagic(builder.setVersion(version).build().toByteArray()); + } + + /** * Verifies current version of file system * * @param fs file system @@ -279,11 +336,11 @@ public abstract class FSUtils { * @param message if true, issues a message on System.out * * @throws IOException e + * @throws DeserializationException */ - public static void checkVersion(FileSystem fs, Path rootdir, - boolean message) throws IOException { - checkVersion(fs, rootdir, message, 0, - HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS); + public static void checkVersion(FileSystem fs, Path rootdir, boolean message) + throws IOException, DeserializationException { + checkVersion(fs, rootdir, message, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS); } /** @@ -296,20 +353,20 @@ public abstract class FSUtils { * @param retries number of times to retry * * @throws IOException e + * @throws DeserializationException */ public static void checkVersion(FileSystem fs, Path rootdir, - boolean message, int wait, int retries) throws IOException { + boolean message, int wait, int retries) + throws IOException, DeserializationException { String version = getVersion(fs, rootdir); - if (version == null) { if (!rootRegionExists(fs, rootdir)) { // rootDir is empty (no version file and no root region) // just create new version file (HBASE-1195) - FSUtils.setVersion(fs, rootdir, wait, retries); + setVersion(fs, rootdir, wait, retries); return; } - } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) - return; + } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) return; // version is deprecated require migration // Output on stdout so user sees it in terminal. @@ -332,8 +389,8 @@ public abstract class FSUtils { */ public static void setVersion(FileSystem fs, Path rootdir) throws IOException { - setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0, - HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS); + setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0, + HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS); } /** @@ -367,19 +424,17 @@ public abstract class FSUtils { while (true) { try { FSDataOutputStream s = fs.create(versionFile); - s.writeUTF(version); - LOG.debug("Created version file at " + rootdir.toString() + - " set its version at:" + version); + s.write(toVersionByteArray(version)); s.close(); + LOG.debug("Created version file at " + rootdir.toString() + " with version=" + version); return; } catch (IOException e) { if (retries > 0) { - LOG.warn("Unable to create version file at " + rootdir.toString() + - ", retrying: " + e.getMessage()); + LOG.warn("Unable to create version file at " + rootdir.toString() + ", retrying", e); fs.delete(versionFile, false); try { if (wait > 0) { - Thread.sleep(wait); + Thread.sleep(wait); } } catch (InterruptedException ex) { // ignore diff --git src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java index 339a120..b732f5a 100644 --- src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java +++ src/test/java/org/apache/hadoop/hbase/util/TestFSUtils.java @@ -21,9 +21,12 @@ package org.apache.hadoop.hbase.util; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.File; +import java.io.IOException; import java.util.UUID; import org.apache.hadoop.conf.Configuration; @@ -32,6 +35,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.DeserializationException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -46,6 +50,30 @@ import org.junit.experimental.categories.Category; */ @Category(MediumTests.class) public class TestFSUtils { + @Test + public void testVersion() throws DeserializationException, IOException { + HBaseTestingUtility htu = new HBaseTestingUtility(); + final FileSystem fs = htu.getTestFileSystem(); + final Path rootdir = htu.getDataTestDir(); + assertNull(FSUtils.getVersion(fs, rootdir)); + // Write out old format version file. See if we can read it in and convert. + Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME); + FSDataOutputStream s = fs.create(versionFile); + final String version = HConstants.FILE_SYSTEM_VERSION; + s.writeUTF(version); + s.close(); + assertTrue(fs.exists(versionFile)); + FileStatus [] status = fs.listStatus(versionFile); + assertNotNull(status); + assertTrue(status.length > 0); + String newVersion = FSUtils.getVersion(fs, rootdir); + assertEquals(version.length(), newVersion.length()); + assertEquals(version, newVersion); + // File will have been converted. Exercise the pb format + assertEquals(version, FSUtils.getVersion(fs, rootdir)); + FSUtils.checkVersion(fs, rootdir, true); + } + @Test public void testIsHDFS() throws Exception { HBaseTestingUtility htu = new HBaseTestingUtility(); htu.getConfiguration().setBoolean("dfs.support.append", false);