diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 25eba7a..a40e042 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -190,7 +190,6 @@ import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.trace.SpanReceiverHost; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CompressionTest; -import org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; @@ -755,9 +754,6 @@ MasterServices, Server { // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring. this.fileSystemManager = new MasterFileSystem(this, this, metricsMaster, masterRecovery); - FSTableDescriptorMigrationToSubdir.migrateFSTableDescriptorsIfNecessary( - this.fileSystemManager.getFileSystem(), this.fileSystemManager.getRootDir()); - this.tableDescriptors = new FSTableDescriptors(this.fileSystemManager.getFileSystem(), this.fileSystemManager.getRootDir()); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java index 2c81719..c6dd11b 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java @@ -478,7 +478,7 @@ public class MasterFileSystem { } // Create tableinfo-s for META if not already there. - new FSTableDescriptors(fs, rd).createTableDescriptor(HTableDescriptor.META_TABLEDESC); + FSTableDescriptors.createTableDescriptor(fs, rd, HTableDescriptor.META_TABLEDESC, false); return rd; } @@ -619,7 +619,7 @@ public class MasterFileSystem { */ public void createTableDescriptor(HTableDescriptor htableDescriptor) throws IOException { - new FSTableDescriptors(conf).createTableDescriptor(htableDescriptor); + FSTableDescriptors.createTableDescriptor(htableDescriptor, conf); } /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java index 677b4d4..97162fd 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java @@ -202,9 +202,8 @@ public class CreateTableHandler extends EventHandler { FileSystem fs = fileSystemManager.getFileSystem(); // 1. Create Table Descriptor + FSTableDescriptors.createTableDescriptor(fs, tempdir, this.hTableDescriptor); Path tempTableDir = new Path(tempdir, tableName); - new FSTableDescriptors(this.conf).createTableDescriptorForTableDirectory( - tempTableDir, this.hTableDescriptor, false); Path tableDir = new Path(fileSystemManager.getRootDir(), tableName); // 2. Create Regions diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java index 58bea29..25f5410 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java @@ -131,7 +131,7 @@ public final class MasterSnapshotVerifier { * @param snapshotDir snapshot directory to check */ private void verifyTableInfo(Path snapshotDir) throws IOException { - FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir); + FSTableDescriptors.getTableDescriptor(fs, snapshotDir); } /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index e2afdf7..f59d1ab 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -667,8 +667,7 @@ public class SnapshotManager implements Stoppable { // read snapshot information SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); - HTableDescriptor snapshotTableDesc = - FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir); + HTableDescriptor snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir); String tableName = reqSnapshot.getTable(); // stop tracking "abandoned" handlers diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java index e0841d3..d40cfb3 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java @@ -51,6 +51,7 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.HRegionInfo; @@ -112,12 +113,12 @@ public class CompactionTool extends Configured implements Tool { if (isFamilyDir(fs, path)) { Path regionDir = path.getParent(); Path tableDir = regionDir.getParent(); - HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); + HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir); HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); compactStoreFiles(tableDir, htd, hri, path.getName(), compactOnce, major); } else if (isRegionDir(fs, path)) { Path tableDir = path.getParent(); - HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); + HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir); compactRegion(tableDir, htd, path, compactOnce, major); } else if (isTableDir(fs, path)) { compactTable(path, compactOnce, major); @@ -129,7 +130,7 @@ public class CompactionTool extends Configured implements Tool { private void compactTable(final Path tableDir, final boolean compactOnce, final boolean major) throws IOException { - HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); + HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir); for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { compactRegion(tableDir, htd, regionDir, compactOnce, major); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java index efcc184..7caad2b 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java @@ -317,7 +317,7 @@ public final class SnapshotInfo extends Configured implements Tool { } snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); - snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir); + snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir); return true; } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java index 16914d2..e7c79e2 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/TableInfoCopyTask.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSTableDescriptors; /** @@ -60,14 +61,12 @@ public class TableInfoCopyTask extends SnapshotTask { LOG.debug("Attempting to copy table info for snapshot:" + ClientSnapshotDescriptionUtils.toString(this.snapshot)); // get the HTable descriptor - - HTableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, - this.snapshot.getTable()); + HTableDescriptor orig = FSTableDescriptors.getTableDescriptor(fs, rootDir, + Bytes.toBytes(this.snapshot.getTable())); this.rethrowException(); // write a copy of descriptor to the snapshot directory Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); - new FSTableDescriptors(fs, rootDir) - .createTableDescriptorForTableDirectory(snapshotDir, orig, false); + FSTableDescriptors.createTableDescriptorForTableDirectory(fs, snapshotDir, orig, false); LOG.debug("Finished copying tableinfo."); return null; } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptorMigrationToSubdir.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptorMigrationToSubdir.java deleted file mode 100644 index 0f978c5..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptorMigrationToSubdir.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; - -/** - * A class to migrate table descriptor files to a dedicated subdir. - * Invoked by HMaster.finishInitialization before accessing table descriptors. - * Migrates snapshots, user tables, and system tables. - * - * @deprecated will be removed for the major release after 0.96. - */ -@Deprecated -public class FSTableDescriptorMigrationToSubdir { - - private static final Log LOG = LogFactory.getLog(FSTableDescriptorMigrationToSubdir.class); - - public static void migrateFSTableDescriptorsIfNecessary(FileSystem fs, Path rootDir) - throws IOException { - if (needsMigration(fs, rootDir)) { - migrateFsTableDescriptors(fs, rootDir); - LOG.info("Migration complete."); - } - } - - /** - * Determines if migration is required by checking to see whether the META table has been - * migrated. - */ - private static boolean needsMigration(FileSystem fs, Path rootDir) throws IOException { - Path metaTableDir = FSTableDescriptors.getTableDirectory(rootDir, - Bytes.toString(HConstants.META_TABLE_NAME)); - FileStatus metaTableInfoStatus = - FSTableDescriptors.getCurrentTableInfoStatus(fs, metaTableDir, false); - return metaTableInfoStatus != null; - } - - /** - * Migrates all snapshots, user tables and system tables that require migration. - * First migrates snapshots. - * Then migrates each user table in order, - * then attempts ROOT (should be gone) - * Migrates META last to indicate migration is complete. - */ - private static void migrateFsTableDescriptors(FileSystem fs, Path rootDir) throws IOException { - // First migrate snapshots - will migrate any snapshot dir that contains a table info file - Path snapshotsDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); - if (fs.exists(snapshotsDir)) { - LOG.info("Migrating snapshots"); - FileStatus[] snapshots = fs.listStatus(snapshotsDir, - new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); - for (FileStatus snapshot : snapshots) { - migrateTable(fs, snapshot.getPath()); - } - } - - LOG.info("Migrating user tables"); - List userTableDirs = FSUtils.getTableDirs(fs, rootDir); - for (Path userTableDir : userTableDirs) { - migrateTable(fs, userTableDir); - } - - LOG.info("Migrating system tables"); - migrateTableIfExists(fs, rootDir, HConstants.ROOT_TABLE_NAME); - // migrate meta last because that's what we check to see if migration is complete - migrateTableIfExists(fs, rootDir, HConstants.META_TABLE_NAME); - } - - private static void migrateTableIfExists(FileSystem fs, Path rootDir, byte[] tableName) - throws IOException { - Path tableDir = FSTableDescriptors.getTableDirectory(rootDir, Bytes.toString(tableName)); - if (fs.exists(tableDir)) { - migrateTable(fs, tableDir); - } - } - - /** - * Migrates table info files. - * Moves the latest table info file (is present) from the table dir to the table info subdir. - * Removes any older table info files from the table dir and any existing table info subdir. - */ - private static void migrateTable(FileSystem fs, Path tableDir) throws IOException { - Path oldTableInfoPath = FSTableDescriptors. - getCurrentTableInfoStatus(fs, tableDir, true).getPath(); - if (oldTableInfoPath == null) { - LOG.debug("No table info file to migrate for " + tableDir); - return; - } - - Path tableInfoDir = new Path(tableDir, FSTableDescriptors.TABLEINFO_DIR); - // remove table info subdir if it already exists - boolean removedExistingSubdir = FSUtils.deleteDirectory(fs, tableInfoDir); - if (removedExistingSubdir) { - LOG.info("Removed existing subdir at: " + tableInfoDir); - } - boolean createdSubdir = fs.mkdirs(tableInfoDir); - if (!createdSubdir) { - throw new IOException("Unable to create new table info directory: " + tableInfoDir); - } - - Path newTableInfoPath = new Path(tableInfoDir, oldTableInfoPath.getName()); - boolean renamedInfoFile = fs.rename(oldTableInfoPath, newTableInfoPath); - if (!renamedInfoFile) { - throw new IOException("Failed to move table info file from old location: " - + oldTableInfoPath + " to new location: " + newTableInfoPath); - } - - LOG.info("Migrated table info from: " + oldTableInfoPath - + " to new location: " + newTableInfoPath); - } - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java index a98bcfd..7f97bee 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.util; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -38,27 +39,25 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableInfoMissingException; -import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Ints; /** * Implementation of {@link TableDescriptors} that reads descriptors from the - * passed filesystem. It expects descriptors to be in a file in the - * {@link #TABLEINFO_DIR} subdir of the table's directory in FS. Can be read-only - * -- i.e. does not modify the filesystem or can be read and write. - * + * passed filesystem. It expects descriptors to be in a file under the + * table's directory in FS. Can be read-only -- i.e. does not modify + * the filesystem or can be read and write. + * *

Also has utility for keeping up the table descriptors tableinfo file. - * The table schema file is kept under the in the {@link #TABLEINFO_DIR} subdir - * of the table directory in the filesystem. - * It has a {@link #TABLEINFO_FILE_PREFIX} and then a suffix that is the + * The table schema file is kept under the table directory in the filesystem. + * It has a {@link #TABLEINFO_NAME} prefix and then a suffix that is the * edit sequenceid: e.g. .tableinfo.0000000003. This sequenceid * is always increasing. It starts at zero. The table schema file with the * highest sequenceid has the most recent schema edit. Usually there is one file @@ -73,29 +72,27 @@ public class FSTableDescriptors implements TableDescriptors { private final FileSystem fs; private final Path rootdir; private final boolean fsreadonly; - @VisibleForTesting long cachehits = 0; - @VisibleForTesting long invocations = 0; + long cachehits = 0; + long invocations = 0; - /** The file name prefix used to store HTD in HDFS */ - static final String TABLEINFO_FILE_PREFIX = ".tableinfo"; - static final String TABLEINFO_DIR = ".tabledesc"; - static final String TMP_DIR = ".tmp"; + /** The file name used to store HTD in HDFS */ + public static final String TABLEINFO_NAME = ".tableinfo"; // This cache does not age out the old stuff. Thinking is that the amount // of data we keep up in here is so small, no need to do occasional purge. // TODO. - private final Map cache = - new ConcurrentHashMap(); + private final Map cache = + new ConcurrentHashMap(); /** * Data structure to hold modification time and table descriptor. */ - private static class TableDescriptorAndModtime { - private final HTableDescriptor htd; + static class TableDescriptorModtime { + private final HTableDescriptor descriptor; private final long modtime; - TableDescriptorAndModtime(final long modtime, final HTableDescriptor htd) { - this.htd = htd; + TableDescriptorModtime(final long modtime, final HTableDescriptor htd) { + this.descriptor = htd; this.modtime = modtime; } @@ -104,40 +101,30 @@ public class FSTableDescriptors implements TableDescriptors { } HTableDescriptor getTableDescriptor() { - return this.htd; + return this.descriptor; } } - /** - * Construct a FSTableDescriptors instance using the hbase root dir of the given - * conf and the filesystem where that root dir lives. - * This instance can do write operations (is not read only). - */ - public FSTableDescriptors(final Configuration conf) throws IOException { - this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf)); - } - public FSTableDescriptors(final FileSystem fs, final Path rootdir) { this(fs, rootdir, false); } /** - * @param fsreadonly True if we are read-only when it comes to filesystem + * @param fs + * @param rootdir + * @param fsreadOnly True if we are read-only when it comes to filesystem * operations; i.e. on remove, we do not do delete in fs. */ - public FSTableDescriptors(final FileSystem fs, - final Path rootdir, final boolean fsreadonly) { + public FSTableDescriptors(final FileSystem fs, final Path rootdir, + final boolean fsreadOnly) { super(); this.fs = fs; this.rootdir = rootdir; - this.fsreadonly = fsreadonly; + this.fsreadonly = fsreadOnly; } - /** - * Get the current table descriptor for the given table, or null if none exists. - * - * Uses a local cache of the descriptor but still checks the filesystem on each call - * to see if a newer file has been created since the cached one was read. + /* (non-Javadoc) + * @see org.apache.hadoop.hbase.TableDescriptors#getHTableDescriptor(java.lang.String) */ @Override public HTableDescriptor get(final byte [] tablename) @@ -145,11 +132,8 @@ public class FSTableDescriptors implements TableDescriptors { return get(Bytes.toString(tablename)); } - /** - * Get the current table descriptor for the given table, or null if none exists. - * - * Uses a local cache of the descriptor but still checks the filesystem on each call - * to see if a newer file has been created since the cached one was read. + /* (non-Javadoc) + * @see org.apache.hadoop.hbase.TableDescriptors#getTableDescriptor(byte[]) */ @Override public HTableDescriptor get(final String tablename) @@ -166,23 +150,23 @@ public class FSTableDescriptors implements TableDescriptors { // .META. and -ROOT- is already handled. If some one tries to get the descriptor for // .logs, .oldlogs or .corrupt throw an exception. if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tablename)) { - throw new IOException("No descriptor found for non table = " + tablename); + throw new IOException("No descriptor found for table = " + tablename); } // Look in cache of descriptors. - TableDescriptorAndModtime cachedtdm = this.cache.get(tablename); + TableDescriptorModtime cachedtdm = this.cache.get(tablename); if (cachedtdm != null) { // Check mod time has not changed (this is trip to NN). - if (getTableInfoModtime(tablename) <= cachedtdm.getModtime()) { + if (getTableInfoModtime(this.fs, this.rootdir, tablename) <= cachedtdm.getModtime()) { cachehits++; return cachedtdm.getTableDescriptor(); } } - - TableDescriptorAndModtime tdmt = null; + + TableDescriptorModtime tdmt = null; try { - tdmt = getTableDescriptorAndModtime(tablename); + tdmt = getTableDescriptorModtime(this.fs, this.rootdir, tablename); } catch (NullPointerException e) { LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, e); @@ -190,7 +174,7 @@ public class FSTableDescriptors implements TableDescriptors { LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, ioe); } - + if (tdmt == null) { LOG.warn("The following folder is in HBase's root directory and " + "doesn't contain a table descriptor, " + @@ -201,8 +185,8 @@ public class FSTableDescriptors implements TableDescriptors { return tdmt == null ? null : tdmt.getTableDescriptor(); } - /** - * Returns a map from table name to table descriptor for all tables. + /* (non-Javadoc) + * @see org.apache.hadoop.hbase.TableDescriptors#getTableDescriptors(org.apache.hadoop.fs.FileSystem, org.apache.hadoop.fs.Path) */ @Override public Map getAll() @@ -224,15 +208,8 @@ public class FSTableDescriptors implements TableDescriptors { return htds; } - /** - * Adds (or updates) the table descriptor to the FileSystem - * and updates the local cache with it. - */ @Override public void add(HTableDescriptor htd) throws IOException { - if (fsreadonly) { - throw new NotImplementedException("Cannot add a table descriptor - in read only mode"); - } if (Bytes.equals(HConstants.ROOT_TABLE_NAME, htd.getName())) { throw new NotImplementedException(); } @@ -240,179 +217,108 @@ public class FSTableDescriptors implements TableDescriptors { throw new NotImplementedException(); } if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(htd.getNameAsString())) { - throw new NotImplementedException( - "Cannot add a table descriptor for a reserved subdirectory name: " + htd.getNameAsString()); + throw new NotImplementedException(); } - updateTableDescriptor(htd); - long modtime = getTableInfoModtime(htd.getNameAsString()); - this.cache.put(htd.getNameAsString(), new TableDescriptorAndModtime(modtime, htd)); + if (!this.fsreadonly) updateHTableDescriptor(this.fs, this.rootdir, htd); + long modtime = getTableInfoModtime(this.fs, this.rootdir, htd.getNameAsString()); + this.cache.put(htd.getNameAsString(), new TableDescriptorModtime(modtime, htd)); } - /** - * Removes the table descriptor from the local cache and returns it. - * If not in read only mode, it also deletes the entire table directory(!) - * from the FileSystem. - */ @Override public HTableDescriptor remove(final String tablename) throws IOException { - if (fsreadonly) { - throw new NotImplementedException("Cannot remove a table descriptor - in read only mode"); - } - Path tabledir = getTableDirectory(tablename); - if (this.fs.exists(tabledir)) { - if (!this.fs.delete(tabledir, true)) { - throw new IOException("Failed delete of " + tabledir.toString()); + if (!this.fsreadonly) { + Path tabledir = FSUtils.getTablePath(this.rootdir, tablename); + if (this.fs.exists(tabledir)) { + if (!this.fs.delete(tabledir, true)) { + throw new IOException("Failed delete of " + tabledir.toString()); + } } } - TableDescriptorAndModtime tdm = this.cache.remove(tablename); + TableDescriptorModtime tdm = this.cache.remove(tablename); return tdm == null ? null : tdm.getTableDescriptor(); } /** - * Checks if a current table info file exists for the given table - * + * Checks if .tableinfo exists for given table + * + * @param fs file system + * @param rootdir root directory of HBase installation * @param tableName name of table * @return true if exists * @throws IOException */ - public boolean isTableInfoExists(String tableName) throws IOException { - return getTableInfoPath(tableName) != null; - } - - /** - * Find the most current table info file for the given table in the hbase root directory. - * @return The file status of the current table info file or null if it does not exist - */ - private FileStatus getTableInfoPath(final String tableName) throws IOException { - Path tableDir = getTableDirectory(tableName); - return getTableInfoPath(tableDir); + public static boolean isTableInfoExists(FileSystem fs, Path rootdir, + String tableName) throws IOException { + FileStatus status = getTableInfoPath(fs, rootdir, tableName); + return status == null? false: fs.exists(status.getPath()); } - private FileStatus getTableInfoPath(Path tableDir) - throws IOException { - return getTableInfoPath(fs, tableDir, !fsreadonly); - } - - /** - * Find the most current table info file for the table located in the given table directory. - * - * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info - * files and takes the 'current' one - meaning the one with the highest sequence number if present - * or no sequence number at all if none exist (for backward compatibility from before there - * were sequence numbers). - * - * @return The file status of the current table info file or null if it does not exist - * @throws IOException - */ - public static FileStatus getTableInfoPath(FileSystem fs, Path tableDir) + private static FileStatus getTableInfoPath(final FileSystem fs, + final Path rootdir, final String tableName) throws IOException { - return getTableInfoPath(fs, tableDir, false); + Path tabledir = FSUtils.getTablePath(rootdir, tableName); + return getTableInfoPath(fs, tabledir); } - - /** - * Find the most current table info file for the table in the given table directory. - * - * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info - * files and takes the 'current' one - meaning the one with the highest sequence number if - * present or no sequence number at all if none exist (for backward compatibility from before - * there were sequence numbers). - * If there are multiple table info files found and removeOldFiles is true it also deletes the - * older files. - * - * @return The file status of the current table info file or null if none exist - * @throws IOException - */ - private static FileStatus getTableInfoPath(FileSystem fs, Path tableDir, boolean removeOldFiles) - throws IOException { - Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR); - return getCurrentTableInfoStatus(fs, tableInfoDir, removeOldFiles); - } - + /** - * Find the most current table info file in the given directory - * - * Looks within the given directory for any table info files - * and takes the 'current' one - meaning the one with the highest sequence number if present - * or no sequence number at all if none exist (for backward compatibility from before there - * were sequence numbers). - * If there are multiple possible files found - * and the we're not in read only mode it also deletes the older files. - * - * @return The file status of the current table info file or null if it does not exist + * Looks under the table directory in the filesystem for files with a + * {@link #TABLEINFO_NAME} prefix. Returns reference to the 'latest' instance. + * @param fs + * @param tabledir + * @return The 'current' tableinfo file. * @throws IOException */ - // only visible for FSTableDescriptorMigrationToSubdir, can be removed with that - static FileStatus getCurrentTableInfoStatus(FileSystem fs, Path dir, boolean removeOldFiles) + public static FileStatus getTableInfoPath(final FileSystem fs, + final Path tabledir) throws IOException { - FileStatus [] status = FSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER); - if (status == null || status.length < 1) return null; - FileStatus mostCurrent = null; - for (FileStatus file : status) { - if (mostCurrent == null || TABLEINFO_FILESTATUS_COMPARATOR.compare(file, mostCurrent) < 0) { - mostCurrent = file; + FileStatus [] status = FSUtils.listStatus(fs, tabledir, new PathFilter() { + @Override + public boolean accept(Path p) { + // Accept any file that starts with TABLEINFO_NAME + return p.getName().startsWith(TABLEINFO_NAME); } - } - if (removeOldFiles && status.length > 1) { - // Clean away old versions - for (FileStatus file : status) { - Path path = file.getPath(); - if (file != mostCurrent) { - if (!fs.delete(file.getPath(), false)) { - LOG.warn("Failed cleanup of " + path); - } else { - LOG.debug("Cleaned up old tableinfo file " + path); - } + }); + if (status == null || status.length < 1) return null; + Arrays.sort(status, new FileStatusFileNameComparator()); + if (status.length > 1) { + // Clean away old versions of .tableinfo + for (int i = 1; i < status.length; i++) { + Path p = status[i].getPath(); + // Clean up old versions + if (!fs.delete(p, false)) { + LOG.warn("Failed cleanup of " + p); + } else { + LOG.debug("Cleaned up old tableinfo file " + p); } } } - return mostCurrent; + return status[0]; } - + /** - * Compare {@link FileStatus} instances by {@link Path#getName()}. Returns in - * reverse order. + * Compare {@link FileStatus} instances by {@link Path#getName()}. + * Returns in reverse order. */ - @VisibleForTesting - static final Comparator TABLEINFO_FILESTATUS_COMPARATOR = - new Comparator() { + static class FileStatusFileNameComparator + implements Comparator { @Override public int compare(FileStatus left, FileStatus right) { return -left.compareTo(right); - }}; - - /** - * Return the table directory in HDFS - */ - @VisibleForTesting Path getTableDirectory(final String tableName) { - return getTableDirectory(rootdir, tableName); - } - - /** - * Return the table directory in HDFS - */ - static Path getTableDirectory(Path rootDir, String tableName) { - return FSUtils.getTablePath(rootDir, tableName); + } } - - private static final PathFilter TABLEINFO_PATHFILTER = new PathFilter() { - @Override - public boolean accept(Path p) { - // Accept any file that starts with TABLEINFO_NAME - return p.getName().startsWith(TABLEINFO_FILE_PREFIX); - }}; /** * Width of the sequenceid that is a suffix on a tableinfo file. */ - @VisibleForTesting static final int WIDTH_OF_SEQUENCE_ID = 10; + static final int WIDTH_OF_SEQUENCE_ID = 10; /* * @param number Number to use as suffix. - * @return Returns zero-prefixed decimal version of passed + * @return Returns zero-prefixed 5-byte wide decimal version of passed * number (Does absolute in case number is negative). */ - private static String formatTableInfoSequenceId(final int number) { + static String formatTableInfoSequenceId(final int number) { byte [] b = new byte[WIDTH_OF_SEQUENCE_ID]; int d = Math.abs(number); for (int i = b.length - 1; i >= 0; i--) { @@ -427,16 +333,17 @@ public class FSTableDescriptors implements TableDescriptors { * Use regex because may encounter oldstyle .tableinfos where there is no * sequenceid on the end. */ - private static final Pattern TABLEINFO_FILE_REGEX = - Pattern.compile(TABLEINFO_FILE_PREFIX + "(\\.([0-9]{" + WIDTH_OF_SEQUENCE_ID + "}))?$"); + private static final Pattern SUFFIX = + Pattern.compile(TABLEINFO_NAME + "(\\.([0-9]{" + WIDTH_OF_SEQUENCE_ID + "}))?$"); + /** * @param p Path to a .tableinfo file. * @return The current editid or 0 if none found. */ - @VisibleForTesting static int getTableInfoSequenceId(final Path p) { + static int getTableInfoSequenceid(final Path p) { if (p == null) return 0; - Matcher m = TABLEINFO_FILE_REGEX.matcher(p.getName()); + Matcher m = SUFFIX.matcher(p.getName()); if (!m.matches()) throw new IllegalArgumentException(p.toString()); String suffix = m.group(2); if (suffix == null || suffix.length() <= 0) return 0; @@ -448,70 +355,73 @@ public class FSTableDescriptors implements TableDescriptors { * @param sequenceid * @return Name of tableinfo file. */ - @VisibleForTesting static String getTableInfoFileName(final int sequenceid) { - return TABLEINFO_FILE_PREFIX + "." + formatTableInfoSequenceId(sequenceid); + static Path getTableInfoFileName(final Path tabledir, final int sequenceid) { + return new Path(tabledir, + TABLEINFO_NAME + "." + formatTableInfoSequenceId(sequenceid)); } /** * @param fs * @param rootdir * @param tableName - * @return Modification time for the table {@link #TABLEINFO_FILE_PREFIX} file + * @return Modification time for the table {@link #TABLEINFO_NAME} file * or 0 if no tableinfo file found. * @throws IOException */ - private long getTableInfoModtime(final String tableName) throws IOException { - FileStatus status = getTableInfoPath(tableName); - return status == null ? 0 : status.getModificationTime(); - } - - /** - * Returns the latest table descriptor for the given table directly from the file system - * if it exists, bypassing the local cache. - * Returns null if it's not found. - */ - public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs, - Path hbaseRootDir, String tableName) throws IOException { - Path tableDir = getTableDirectory(hbaseRootDir, tableName); - return getTableDescriptorFromFs(fs, tableDir); + static long getTableInfoModtime(final FileSystem fs, final Path rootdir, + final String tableName) + throws IOException { + FileStatus status = getTableInfoPath(fs, rootdir, tableName); + return status == null? 0: status.getModificationTime(); } /** - * Returns the latest table descriptor for the table located at the given directory - * directly from the file system if it exists. - * @throws TableInfoMissingException if there is no descriptor + * Get HTD from HDFS. + * @param fs + * @param hbaseRootDir + * @param tableName + * @return Descriptor or null if none found. + * @throws IOException */ - public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir) + public static HTableDescriptor getTableDescriptor(FileSystem fs, + Path hbaseRootDir, byte[] tableName) throws IOException { - FileStatus status = getTableInfoPath(fs, tableDir, false); - if (status == null) { - throw new TableInfoMissingException("No table descriptor file under " + tableDir); - } - return readTableDescriptor(fs, status, false); + HTableDescriptor htd = null; + try { + TableDescriptorModtime tdmt = + getTableDescriptorModtime(fs, hbaseRootDir, Bytes.toString(tableName)); + htd = tdmt == null ? null : tdmt.getTableDescriptor(); + } catch (NullPointerException e) { + LOG.debug("Exception during readTableDecriptor. Current table name = " + + Bytes.toString(tableName), e); + } + return htd; + } + + static HTableDescriptor getTableDescriptor(FileSystem fs, + Path hbaseRootDir, String tableName) throws NullPointerException, IOException { + TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, hbaseRootDir, tableName); + return tdmt == null ? null : tdmt.getTableDescriptor(); } - - private TableDescriptorAndModtime getTableDescriptorAndModtime(String tableName) - throws IOException { + + static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, + Path hbaseRootDir, String tableName) throws NullPointerException, IOException{ // ignore both -ROOT- and .META. tables if (Bytes.compareTo(Bytes.toBytes(tableName), HConstants.ROOT_TABLE_NAME) == 0 || Bytes.compareTo(Bytes.toBytes(tableName), HConstants.META_TABLE_NAME) == 0) { return null; } - return getTableDescriptorAndModtime(getTableDirectory(tableName)); + return getTableDescriptorModtime(fs, FSUtils.getTablePath(hbaseRootDir, tableName)); } - private TableDescriptorAndModtime getTableDescriptorAndModtime(Path tableDir) - throws IOException { - FileStatus status = getTableInfoPath(tableDir); + static TableDescriptorModtime getTableDescriptorModtime(FileSystem fs, Path tableDir) + throws NullPointerException, IOException { + if (tableDir == null) throw new NullPointerException(); + FileStatus status = getTableInfoPath(fs, tableDir); if (status == null) { - throw new TableInfoMissingException("No table descriptor file under " + tableDir); + throw new TableInfoMissingException("No .tableinfo file under " + + tableDir.toUri()); } - HTableDescriptor htd = readTableDescriptor(fs, status, !fsreadonly); - return new TableDescriptorAndModtime(status.getModificationTime(), htd); - } - - private static HTableDescriptor readTableDescriptor(FileSystem fs, FileStatus status, - boolean rewritePb) throws IOException { int len = Ints.checkedCast(status.getLen()); byte [] content = new byte[len]; FSDataInputStream fsDataInputStream = fs.open(status.getPath()); @@ -526,133 +436,110 @@ public class FSTableDescriptors implements TableDescriptors { } catch (DeserializationException e) { throw new IOException("content=" + Bytes.toShort(content), e); } - if (rewritePb && !ProtobufUtil.isPBMagicPrefix(content)) { + if (!ProtobufUtil.isPBMagicPrefix(content)) { // Convert the file over to be pb before leaving here. - Path tableInfoDir = status.getPath().getParent(); - Path tableDir = tableInfoDir.getParent(); - writeTableDescriptor(fs, htd, tableDir, status); + createTableDescriptor(fs, tableDir.getParent(), htd, true); } - return htd; + return new TableDescriptorModtime(status.getModificationTime(), htd); + } + + public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir) + throws IOException, NullPointerException { + TableDescriptorModtime tdmt = getTableDescriptorModtime(fs, tableDir); + return tdmt == null ? null : tdmt.getTableDescriptor(); } - + /** - * Update table descriptor on the file system + * Update table descriptor + * @param fs + * @param conf + * @param hTableDescriptor + * @return New tableinfo or null if we failed update. * @throws IOException Thrown if failed update. - * @throws NotImplementedException if in read only mode */ - @VisibleForTesting Path updateTableDescriptor(HTableDescriptor htd) + static Path updateHTableDescriptor(FileSystem fs, Path rootdir, + HTableDescriptor hTableDescriptor) throws IOException { - if (fsreadonly) { - throw new NotImplementedException("Cannot update a table descriptor - in read only mode"); - } - Path tableDir = getTableDirectory(htd.getNameAsString()); - Path p = writeTableDescriptor(fs, htd, tableDir, getTableInfoPath(tableDir)); + Path tableDir = FSUtils.getTablePath(rootdir, hTableDescriptor.getName()); + Path p = writeTableDescriptor(fs, hTableDescriptor, tableDir, + getTableInfoPath(fs, tableDir)); if (p == null) throw new IOException("Failed update"); LOG.info("Updated tableinfo=" + p); return p; } /** - * Deletes all the table descriptor files from the file system. - * Used in unit tests only. - * @throws NotImplementedException if in read only mode + * Deletes a table's directory from the file system if exists. Used in unit + * tests. */ - public void deleteTableDescriptorIfExists(String tableName) throws IOException { - if (fsreadonly) { - throw new NotImplementedException("Cannot delete a table descriptor - in read only mode"); + public static void deleteTableDescriptorIfExists(String tableName, + Configuration conf) throws IOException { + FileSystem fs = FSUtils.getCurrentFileSystem(conf); + FileStatus status = getTableInfoPath(fs, FSUtils.getRootDir(conf), tableName); + // The below deleteDirectory works for either file or directory. + if (status != null && fs.exists(status.getPath())) { + FSUtils.deleteDirectory(fs, status.getPath()); } - - Path tableDir = getTableDirectory(tableName); - Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR); - deleteTableDescriptorFiles(fs, tableInfoDir, Integer.MAX_VALUE); } /** - * Deletes files matching the table info file pattern within the given directory - * whose sequenceId is at most the given max sequenceId. - */ - private static void deleteTableDescriptorFiles(FileSystem fs, Path dir, int maxSequenceId) - throws IOException { - FileStatus [] status = FSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER); - for (FileStatus file : status) { - Path path = file.getPath(); - int sequenceId = getTableInfoSequenceId(path); - if (sequenceId <= maxSequenceId) { - boolean success = FSUtils.delete(fs, path, false); - if (success) { - LOG.debug("Deleted table descriptor at " + path); - } else { - LOG.error("Failed to delete descriptor at " + path); - } - } - } - } - - /** - * Attempts to write a new table descriptor to the given table's directory. - * It first writes it to the .tmp dir then uses an atomic rename to move it into place. - * It begins at the currentSequenceId + 1 and tries 10 times to find a new sequence number - * not already in use. - * Removes the current descriptor file if passed in. - * + * @param fs + * @param hTableDescriptor + * @param tableDir + * @param status * @return Descriptor file or null if we failed write. + * @throws IOException */ - private static Path writeTableDescriptor(final FileSystem fs, - final HTableDescriptor htd, final Path tableDir, - final FileStatus currentDescriptorFile) - throws IOException { + private static Path writeTableDescriptor(final FileSystem fs, + final HTableDescriptor hTableDescriptor, final Path tableDir, + final FileStatus status) + throws IOException { // Get temporary dir into which we'll first write a file to avoid half-written file phenomenon. - // This directory is never removed to avoid removing it out from under a concurrent writer. - Path tmpTableDir = new Path(tableDir, TMP_DIR); - Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR); - + Path tmpTableDir = new Path(tableDir, ".tmp"); // What is current sequenceid? We read the current sequenceid from // the current file. After we read it, another thread could come in and // compete with us writing out next version of file. The below retries // should help in this case some but its hard to do guarantees in face of // concurrent schema edits. - int currentSequenceId = currentDescriptorFile == null ? 0 : - getTableInfoSequenceId(currentDescriptorFile.getPath()); - int newSequenceId = currentSequenceId; - + int currentSequenceid = status == null? 0: getTableInfoSequenceid(status.getPath()); + int sequenceid = currentSequenceid; // Put arbitrary upperbound on how often we retry int retries = 10; - int retrymax = currentSequenceId + retries; - Path tableInfoDirPath = null; + int retrymax = currentSequenceid + retries; + Path tableInfoPath = null; do { - newSequenceId += 1; - String filename = getTableInfoFileName(newSequenceId); - Path tempPath = new Path(tmpTableDir, filename); - if (fs.exists(tempPath)) { - LOG.debug(tempPath + " exists; retrying up to " + retries + " times"); + sequenceid += 1; + Path p = getTableInfoFileName(tmpTableDir, sequenceid); + if (fs.exists(p)) { + LOG.debug(p + " exists; retrying up to " + retries + " times"); continue; } - tableInfoDirPath = new Path(tableInfoDir, filename); try { - writeHTD(fs, tempPath, htd); - fs.mkdirs(tableInfoDirPath.getParent()); - if (!fs.rename(tempPath, tableInfoDirPath)) { - throw new IOException("Failed rename of " + tempPath + " to " + tableInfoDirPath); + writeHTD(fs, p, hTableDescriptor); + tableInfoPath = getTableInfoFileName(tableDir, sequenceid); + if (!fs.rename(p, tableInfoPath)) { + throw new IOException("Failed rename of " + p + " to " + tableInfoPath); } - LOG.debug("Wrote descriptor into: " + tableInfoDirPath); } catch (IOException ioe) { // Presume clash of names or something; go around again. LOG.debug("Failed write and/or rename; retrying", ioe); - if (!FSUtils.deleteDirectory(fs, tempPath)) { - LOG.warn("Failed cleanup of " + tempPath); + if (!FSUtils.deleteDirectory(fs, p)) { + LOG.warn("Failed cleanup of " + p); } - tableInfoDirPath = null; + tableInfoPath = null; continue; } + // Cleanup old schema file. + if (status != null) { + if (!FSUtils.deleteDirectory(fs, status.getPath())) { + LOG.warn("Failed delete of " + status.getPath() + "; continuing"); + } + } break; - } while (newSequenceId < retrymax); - if (tableInfoDirPath != null) { - // if we succeeded, remove old table info files. - deleteTableDescriptorFiles(fs, tableInfoDir, newSequenceId - 1); - } - return tableInfoDirPath; + } while (sequenceid < retrymax); + return tableInfoPath; } - + private static void writeHTD(final FileSystem fs, final Path p, final HTableDescriptor htd) throws IOException { FSDataOutputStream out = fs.create(p, false); @@ -667,57 +554,90 @@ public class FSTableDescriptors implements TableDescriptors { /** * Create new HTableDescriptor in HDFS. Happens when we are creating table. + * + * @param htableDescriptor + * @param conf + */ + public static boolean createTableDescriptor(final HTableDescriptor htableDescriptor, + Configuration conf) + throws IOException { + return createTableDescriptor(htableDescriptor, conf, false); + } + + /** + * Create new HTableDescriptor in HDFS. Happens when we are creating table. If + * forceCreation is true then even if previous table descriptor is present it + * will be overwritten + * + * @param htableDescriptor + * @param conf + * @param forceCreation True if we are to overwrite existing file. + */ + static boolean createTableDescriptor(final HTableDescriptor htableDescriptor, + final Configuration conf, boolean forceCreation) + throws IOException { + FileSystem fs = FSUtils.getCurrentFileSystem(conf); + return createTableDescriptor(fs, FSUtils.getRootDir(conf), htableDescriptor, forceCreation); + } + + /** + * Create new HTableDescriptor in HDFS. Happens when we are creating table. * Used by tests. - * @return True if we successfully created file. + * @param fs + * @param htableDescriptor + * @param rootdir */ - public boolean createTableDescriptor(HTableDescriptor htd) throws IOException { - return createTableDescriptor(htd, false); + public static boolean createTableDescriptor(FileSystem fs, Path rootdir, + HTableDescriptor htableDescriptor) + throws IOException { + return createTableDescriptor(fs, rootdir, htableDescriptor, false); } /** * Create new HTableDescriptor in HDFS. Happens when we are creating table. If * forceCreation is true then even if previous table descriptor is present it * will be overwritten - * + * + * @param fs + * @param htableDescriptor + * @param rootdir + * @param forceCreation * @return True if we successfully created file. */ - public boolean createTableDescriptor(HTableDescriptor htd, boolean forceCreation) + public static boolean createTableDescriptor(FileSystem fs, Path rootdir, + HTableDescriptor htableDescriptor, boolean forceCreation) throws IOException { - Path tableDir = getTableDirectory(htd.getNameAsString()); - return createTableDescriptorForTableDirectory(tableDir, htd, forceCreation); + Path tabledir = FSUtils.getTablePath(rootdir, htableDescriptor.getNameAsString()); + return createTableDescriptorForTableDirectory(fs, tabledir, htableDescriptor, forceCreation); } - + /** * Create a new HTableDescriptor in HDFS in the specified table directory. Happens when we create * a new table or snapshot a table. - * @param tableDir table directory under which we should write the file - * @param htd description of the table to write + * @param fs filesystem where the descriptor should be written + * @param tabledir directory under which we should write the file + * @param htableDescriptor description of the table to write * @param forceCreation if true,then even if previous table descriptor is present it will * be overwritten * @return true if the we successfully created the file, false if the file * already exists and we weren't forcing the descriptor creation. * @throws IOException if a filesystem error occurs */ - public boolean createTableDescriptorForTableDirectory(Path tableDir, - HTableDescriptor htd, boolean forceCreation) throws IOException { - if (fsreadonly) { - throw new NotImplementedException("Cannot create a table descriptor - in read only mode"); - } - FileStatus status = getTableInfoPath(fs, tableDir); + public static boolean createTableDescriptorForTableDirectory(FileSystem fs, Path tabledir, + HTableDescriptor htableDescriptor, boolean forceCreation) throws IOException { + FileStatus status = getTableInfoPath(fs, tabledir); if (status != null) { - LOG.debug("Current tableInfoPath = " + status.getPath()); + LOG.info("Current tableInfoPath = " + status.getPath()); if (!forceCreation) { if (fs.exists(status.getPath()) && status.getLen() > 0) { - if (readTableDescriptor(fs, status, false).equals(htd)) { - LOG.debug("TableInfo already exists.. Skipping creation"); + if (getTableDescriptor(fs, status.getPath().getParent()).equals(htableDescriptor)) { + LOG.info("TableInfo already exists.. Skipping creation"); return false; } } } } - Path p = writeTableDescriptor(fs, htd, tableDir, status); + Path p = writeTableDescriptor(fs, htableDescriptor, tabledir, status); return p != null; } - } - diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 0d6ece4..2900a14 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -804,7 +804,7 @@ public class HBaseFsck extends Configured implements Tool { tablesInfo.put(tableName, modTInfo); try { HTableDescriptor htd = - FSTableDescriptors.getTableDescriptorFromFs(hbaseRoot.getFileSystem(getConf()), + FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(getConf()), hbaseRoot, tableName); modTInfo.htds.add(htd); } catch (IOException ioe) { @@ -849,16 +849,16 @@ public class HBaseFsck extends Configured implements Tool { * 1. the correct tablename
* 2. the correct colfamily list
* 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}
+ * @param tableName * @throws IOException */ - private boolean fabricateTableInfo(FSTableDescriptors fstd, String tableName, - Set columns) throws IOException { + private boolean fabricateTableInfo(String tableName, Set columns) throws IOException { if (columns ==null || columns.isEmpty()) return false; HTableDescriptor htd = new HTableDescriptor(tableName); for (String columnfamimly : columns) { htd.addFamily(new HColumnDescriptor(columnfamimly)); } - fstd.createTableDescriptor(htd, true); + FSTableDescriptors.createTableDescriptor(htd, getConf(), true); return true; } @@ -889,13 +889,13 @@ public class HBaseFsck extends Configured implements Tool { public void fixOrphanTables() throws IOException { if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) { + Path hbaseRoot = FSUtils.getRootDir(getConf()); List tmpList = new ArrayList(); tmpList.addAll(orphanTableDirs.keySet()); HTableDescriptor[] htds = getHTableDescriptors(tmpList); Iterator>> iter = orphanTableDirs.entrySet().iterator(); int j = 0; int numFailedCase = 0; - FSTableDescriptors fstd = new FSTableDescriptors(getConf()); while (iter.hasNext()) { Entry> entry = (Entry>) iter.next(); String tableName = entry.getKey(); @@ -904,12 +904,13 @@ public class HBaseFsck extends Configured implements Tool { if (tableName.equals(Bytes.toString(htds[j].getName()))) { HTableDescriptor htd = htds[j]; LOG.info("fixing orphan table: " + tableName + " from cache"); - fstd.createTableDescriptor(htd, true); + FSTableDescriptors.createTableDescriptor( + hbaseRoot.getFileSystem(getConf()), hbaseRoot, htd, true); j++; iter.remove(); } } else { - if (fabricateTableInfo(fstd, tableName, entry.getValue())) { + if (fabricateTableInfo(tableName, entry.getValue())) { LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file"); LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName); iter.remove(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java index 5a43370..00550ed 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.RemoteExceptionHandler; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.catalog.MetaEditor; @@ -42,9 +43,12 @@ import org.apache.hadoop.hbase.client.HConnectable; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLogFactory; @@ -149,7 +153,7 @@ class HMerge { this.rootDir = FSUtils.getRootDir(conf); Path tabledir = HTableDescriptor.getTableDir(this.rootDir, tableName); - this.htd = FSTableDescriptors.getTableDescriptorFromFs(this.fs, tabledir); + this.htd = FSTableDescriptors.getTableDescriptor(this.fs, tabledir); String logname = "merge_" + System.currentTimeMillis() + HConstants.HREGION_LOGDIR_NAME; this.hlog = HLogFactory.createHLog(fs, tabledir, logname, conf); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java index 3ade888..de8adcd 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java @@ -61,6 +61,7 @@ public class Merge extends Configured implements Tool { private byte [] tableName; // Name of table private volatile byte [] region1; // Name of region 1 private volatile byte [] region2; // Name of region 2 + private volatile boolean isMetaTable; private volatile HRegionInfo mergeInfo; /** default constructor */ @@ -152,8 +153,8 @@ public class Merge extends Configured implements Tool { if (info2 == null) { throw new NullPointerException("info2 is null using key " + meta); } - HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(FileSystem.get(getConf()), - this.rootdir, Bytes.toString(this.tableName)); + HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(FileSystem.get(getConf()), + this.rootdir, this.tableName); HRegion merged = merge(htd, meta, info1, info2); LOG.info("Adding " + merged.getRegionInfo() + " to " + @@ -245,6 +246,7 @@ public class Merge extends Configured implements Tool { return -1; } tableName = Bytes.toBytes(remainingArgs[0]); + isMetaTable = Bytes.compareTo(tableName, HConstants.META_TABLE_NAME) == 0; region1 = Bytes.toBytesBinary(remainingArgs[1]); region2 = Bytes.toBytesBinary(remainingArgs[2]); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java hbase-server/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java index 9381a12..91bbba9 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/TestFSTableDescriptorForceCreation.java @@ -39,10 +39,10 @@ public class TestFSTableDescriptorForceCreation { final String name = "newTable2"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path rootdir = new Path(UTIL.getDataTestDir(), name); - FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir); HTableDescriptor htd = new HTableDescriptor(name); - assertTrue("Should create new table descriptor", fstd.createTableDescriptor(htd, false)); + assertTrue("Should create new table descriptor", + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd, false)); } @Test @@ -50,12 +50,13 @@ public class TestFSTableDescriptorForceCreation { throws IOException { final String name = "testAlreadyExists"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - // Cleanup old tests if any detritus laying around. + // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); - FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir); + TableDescriptors htds = new FSTableDescriptors(fs, rootdir); HTableDescriptor htd = new HTableDescriptor(name); - fstd.add(htd); - assertFalse("Should not create new table descriptor", fstd.createTableDescriptor(htd, false)); + htds.add(htd); + assertFalse("Should not create new table descriptor", + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd, false)); } @Test @@ -64,11 +65,10 @@ public class TestFSTableDescriptorForceCreation { final String name = "createNewTableNew2"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path rootdir = new Path(UTIL.getDataTestDir(), name); - FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir); HTableDescriptor htd = new HTableDescriptor(name); - fstd.createTableDescriptor(htd, false); + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd, false); assertTrue("Should create new table descriptor", - fstd.createTableDescriptor(htd, true)); + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd, true)); } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 8efb9c2..d815bc2 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -193,9 +193,8 @@ public class TestMasterFailover { FileSystem filesystem = FileSystem.get(conf); Path rootdir = FSUtils.getRootDir(conf); - FSTableDescriptors fstd = new FSTableDescriptors(filesystem, rootdir); // Write the .tableinfo - fstd.createTableDescriptor(htdEnabled); + FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled); HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(), null, null); createRegion(hriEnabled, rootdir, conf, htdEnabled); @@ -207,7 +206,7 @@ public class TestMasterFailover { HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable); htdDisabled.addFamily(new HColumnDescriptor(FAMILY)); // Write the .tableinfo - fstd.createTableDescriptor(htdDisabled); + FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled); HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null); createRegion(hriDisabled, rootdir, conf, htdDisabled); List disabledRegions = TEST_UTIL.createMultiRegionsInMeta( @@ -496,9 +495,8 @@ public class TestMasterFailover { htdEnabled.addFamily(new HColumnDescriptor(FAMILY)); FileSystem filesystem = FileSystem.get(conf); Path rootdir = FSUtils.getRootDir(conf); - FSTableDescriptors fstd = new FSTableDescriptors(filesystem, rootdir); // Write the .tableinfo - fstd.createTableDescriptor(htdEnabled); + FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled); HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(), null, null); createRegion(hriEnabled, rootdir, conf, htdEnabled); @@ -510,7 +508,7 @@ public class TestMasterFailover { HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable); htdDisabled.addFamily(new HColumnDescriptor(FAMILY)); // Write the .tableinfo - fstd.createTableDescriptor(htdDisabled); + FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled); HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null); createRegion(hriDisabled, rootdir, conf, htdDisabled); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java index e431a2f..8b6e8b9 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDescriptorModification.java @@ -144,7 +144,7 @@ public class TestTableDescriptorModification { // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = HTableDescriptor.getTableDir(mfs.getRootDir(), tableName); - htd = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); + htd = FSTableDescriptors.getTableDescriptor(mfs.getFileSystem(), tableDir); verifyTableDescriptor(htd, tableName, families); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index b1cabbe..39b3c26 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -185,9 +185,9 @@ public class SnapshotTestingUtils { HConstants.HREGION_LOGDIR_NAME)); } // check the table info - HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, - Bytes.toString(tableName)); - HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, + HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, + tableName); + HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir); assertEquals(desc, snapshotDesc); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index 433ffb4..94f9c85 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -376,10 +376,9 @@ public class TestFlushSnapshotFromClient { assertTrue(fs.exists(snapshotinfo)); // check the table info - HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, - rootDir,STRING_TABLE_NAME); - HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, - SnapshotDescriptionUtils.getSnapshotsDir(rootDir), snapshotName); + HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, TABLE_NAME); + HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs, + SnapshotDescriptionUtils.getSnapshotsDir(rootDir), Bytes.toBytes(snapshotName)); assertEquals(desc, snapshotDesc); // check the region snapshot for all the regions diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java index 8c2f384..087f108 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java @@ -126,7 +126,7 @@ public class TestRestoreSnapshotHelper { LOG.debug("pre-restore table=" + htdClone.getNameAsString() + " snapshot=" + snapshotDir); FSUtils.logFileSystemState(fs, rootDir, LOG); - new FSTableDescriptors(conf).createTableDescriptor(htdClone); + FSTableDescriptors.createTableDescriptor(htdClone, conf); RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone); helper.restoreHdfsRegions(); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java index 3d4caf5..8412bfe 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java @@ -27,7 +27,6 @@ import static org.junit.Assert.fail; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; -import java.util.Comparator; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -56,12 +55,12 @@ public class TestFSTableDescriptors { @Test (expected=IllegalArgumentException.class) public void testRegexAgainstOldStyleTableInfo() { - Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX); - int i = FSTableDescriptors.getTableInfoSequenceId(p); + Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_NAME); + int i = FSTableDescriptors.getTableInfoSequenceid(p); assertEquals(0, i); // Assert it won't eat garbage -- that it fails p = new Path("/tmp", "abc"); - FSTableDescriptors.getTableInfoSequenceId(p); + FSTableDescriptors.getTableInfoSequenceid(p); } @Test @@ -69,13 +68,12 @@ public class TestFSTableDescriptors { Path testdir = UTIL.getDataTestDir("testCreateAndUpdate"); HTableDescriptor htd = new HTableDescriptor("testCreate"); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir); - assertTrue(fstd.createTableDescriptor(htd)); - assertFalse(fstd.createTableDescriptor(htd)); + assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); + assertFalse(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); FileStatus [] statuses = fs.listStatus(testdir); assertTrue("statuses.length="+statuses.length, statuses.length == 1); for (int i = 0; i < 10; i++) { - fstd.updateTableDescriptor(htd); + FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); } statuses = fs.listStatus(testdir); assertTrue(statuses.length == 1); @@ -85,53 +83,53 @@ public class TestFSTableDescriptors { } @Test - public void testSequenceIdAdvancesOnTableInfo() throws IOException { + public void testSequenceidAdvancesOnTableInfo() throws IOException { Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo"); HTableDescriptor htd = new HTableDescriptor("testSequenceidAdvancesOnTableInfo"); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir); - Path p0 = fstd.updateTableDescriptor(htd); - int i0 = FSTableDescriptors.getTableInfoSequenceId(p0); - Path p1 = fstd.updateTableDescriptor(htd); + Path p0 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); + int i0 = FSTableDescriptors.getTableInfoSequenceid(p0); + Path p1 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); // Assert we cleaned up the old file. assertTrue(!fs.exists(p0)); - int i1 = FSTableDescriptors.getTableInfoSequenceId(p1); + int i1 = FSTableDescriptors.getTableInfoSequenceid(p1); assertTrue(i1 == i0 + 1); - Path p2 = fstd.updateTableDescriptor(htd); + Path p2 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); // Assert we cleaned up the old file. assertTrue(!fs.exists(p1)); - int i2 = FSTableDescriptors.getTableInfoSequenceId(p2); + int i2 = FSTableDescriptors.getTableInfoSequenceid(p2); assertTrue(i2 == i1 + 1); } @Test public void testFormatTableInfoSequenceId() { - Path p0 = assertWriteAndReadSequenceId(0); + Path p0 = assertWriteAndReadSequenceid(0); // Assert p0 has format we expect. StringBuilder sb = new StringBuilder(); for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) { sb.append("0"); } - assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(), + assertEquals(FSTableDescriptors.TABLEINFO_NAME + "." + sb.toString(), p0.getName()); // Check a few more. - Path p2 = assertWriteAndReadSequenceId(2); - Path p10000 = assertWriteAndReadSequenceId(10000); + Path p2 = assertWriteAndReadSequenceid(2); + Path p10000 = assertWriteAndReadSequenceid(10000); // Get a .tablinfo that has no sequenceid suffix. - Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX); + Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_NAME); FileStatus fs = new FileStatus(0, false, 0, 0, 0, p); FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0); FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2); FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000); - Comparator comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR; + FSTableDescriptors.FileStatusFileNameComparator comparator = + new FSTableDescriptors.FileStatusFileNameComparator(); assertTrue(comparator.compare(fs, fs0) > 0); assertTrue(comparator.compare(fs0, fs2) > 0); assertTrue(comparator.compare(fs2, fs10000) > 0); } - private Path assertWriteAndReadSequenceId(final int i) { - Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i)); - int ii = FSTableDescriptors.getTableInfoSequenceId(p); + private Path assertWriteAndReadSequenceid(final int i) { + Path p = FSTableDescriptors.getTableInfoFileName(new Path("/tmp"), i); + int ii = FSTableDescriptors.getTableInfoSequenceid(p); assertEquals(i, ii); return p; } @@ -154,19 +152,30 @@ public class TestFSTableDescriptors { FileSystem fs = FileSystem.get(UTIL.getConfiguration()); HTableDescriptor htd = new HTableDescriptor(name); Path rootdir = UTIL.getDataTestDir(name); - FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir); - fstd.createTableDescriptor(htd); + createHTDInFS(fs, rootdir, htd); HTableDescriptor htd2 = - FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getNameAsString()); + FSTableDescriptors.getTableDescriptor(fs, rootdir, htd.getNameAsString()); assertTrue(htd.equals(htd2)); } + private void createHTDInFS(final FileSystem fs, Path rootdir, + final HTableDescriptor htd) + throws IOException { + FSTableDescriptors.createTableDescriptor(fs, rootdir, htd); + } + @Test public void testHTableDescriptors() throws IOException, InterruptedException { final String name = "testHTableDescriptors"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any debris laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); + final int count = 10; + // Write out table infos. + for (int i = 0; i < count; i++) { + HTableDescriptor htd = new HTableDescriptor(name + i); + createHTDInFS(fs, rootdir, htd); + } FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) { @Override public HTableDescriptor get(byte[] tablename) @@ -175,13 +184,6 @@ public class TestFSTableDescriptors { return super.get(tablename); } }; - final int count = 10; - // Write out table infos. - for (int i = 0; i < count; i++) { - HTableDescriptor htd = new HTableDescriptor(name + i); - htds.createTableDescriptor(htd); - } - for (int i = 0; i < count; i++) { assertTrue(htds.get(Bytes.toBytes(name + i)) != null); } @@ -192,7 +194,7 @@ public class TestFSTableDescriptors { for (int i = 0; i < count; i++) { HTableDescriptor htd = new HTableDescriptor(name + i); htd.addFamily(new HColumnDescriptor("" + i)); - htds.updateTableDescriptor(htd); + FSTableDescriptors.updateHTableDescriptor(fs, rootdir, htd); } // Wait a while so mod time we write is for sure different. Thread.sleep(100); @@ -238,8 +240,7 @@ public class TestFSTableDescriptors { @Test public void testTableInfoFileStatusComparator() { FileStatus bare = - new FileStatus(0, false, 0, 0, -1, - new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX)); + new FileStatus(0, false, 0, 0, -1, new Path("/tmp", FSTableDescriptors.TABLEINFO_NAME)); FileStatus future = new FileStatus(0, false, 0, 0, -1, new Path("/tmp/tablinfo." + System.currentTimeMillis())); @@ -249,7 +250,8 @@ public class TestFSTableDescriptors { FileStatus [] alist = {bare, future, farFuture}; FileStatus [] blist = {bare, farFuture, future}; FileStatus [] clist = {farFuture, bare, future}; - Comparator c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR; + FSTableDescriptors.FileStatusFileNameComparator c = + new FSTableDescriptors.FileStatusFileNameComparator(); Arrays.sort(alist, c); Arrays.sort(blist, c); Arrays.sort(clist, c); @@ -280,17 +282,16 @@ public class TestFSTableDescriptors { HTableDescriptor htd = new HTableDescriptor( "testCreateTableDescriptorUpdatesIfThereExistsAlready"); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir); - assertTrue(fstd.createTableDescriptor(htd)); - assertFalse(fstd.createTableDescriptor(htd)); + assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); + assertFalse(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); htd.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue")); - assertTrue(fstd.createTableDescriptor(htd)); //this will re-create - Path tableDir = fstd.getTableDirectory(htd.getNameAsString()); - Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR); + assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); //this will re-create + Path tableDir = FSUtils.getTablePath(testdir, htd.getName()); + Path tmpTableDir = new Path(tableDir, ".tmp"); FileStatus[] statuses = fs.listStatus(tmpTableDir); assertTrue(statuses.length == 0); - assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir)); + assertEquals(htd, FSTableDescriptors.getTableDescriptor(fs, tableDir)); } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index 6a35480..d22771f 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -1099,6 +1099,8 @@ public class TestHBaseFsck { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); assertTrue(cluster.waitForActiveAndReadyMaster()); + FileSystem filesystem = FileSystem.get(conf); + Path rootdir = FSUtils.getRootDir(conf); byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") }; @@ -1106,8 +1108,8 @@ public class TestHBaseFsck { htdDisabled.addFamily(new HColumnDescriptor(FAM)); // Write the .tableinfo - FSTableDescriptors fstd = new FSTableDescriptors(conf); - fstd.createTableDescriptor(htdDisabled); + FSTableDescriptors + .createTableDescriptor(filesystem, rootdir, htdDisabled); List disabledRegions = TEST_UTIL.createMultiRegionsInMeta( TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java index 7b192c3..e8ff65b 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java @@ -95,7 +95,7 @@ public class TestMergeTable { // Create regions and populate them at same time. Create the tabledir // for them first. - new FSTableDescriptors(fs, rootdir).createTableDescriptor(desc); + FSTableDescriptors.createTableDescriptor(fs, rootdir, desc); HRegion [] regions = { createRegion(desc, null, row_70001, 1, 70000, rootdir), createRegion(desc, row_70001, row_80001, 70001, 10000, rootdir), diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java index c234319..187ed07 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java @@ -139,7 +139,7 @@ public class TestMergeTool extends HBaseTestCase { try { // Create meta region createMetaRegion(); - new FSTableDescriptors(this.fs, this.testDir).createTableDescriptor(this.desc); + FSTableDescriptors.createTableDescriptor(this.fs, this.testDir, this.desc); /* * Create the regions we will merge */