From 6bbbca705795ee45b0641733c38e50ddc37e0a9c Mon Sep 17 00:00:00 2001 From: Umesh Agashe Date: Fri, 21 Oct 2016 14:28:17 -0700 Subject: [PATCH] HBASE-16957 Moved cleaners from master package to fs.legacy package and removed filesystem/ directory layout references from a few files in master package --- hbase-common/src/main/resources/hbase-default.xml | 30 +- .../example/LongTermArchivingHFileCleaner.java | 4 +- .../org/apache/hadoop/hbase/fs/MasterStorage.java | 90 ++++- .../hadoop/hbase/fs/legacy/LegacyLayout.java | 4 + .../hbase/fs/legacy/LegacyMasterStorage.java | 113 +++++- .../fs/legacy/cleaner/BaseFileCleanerDelegate.java | 56 +++ .../legacy/cleaner/BaseHFileCleanerDelegate.java | 54 +++ .../fs/legacy/cleaner/BaseLogCleanerDelegate.java | 57 +++ .../hbase/fs/legacy/cleaner/CleanerChore.java | 294 +++++++++++++++ .../fs/legacy/cleaner/FileCleanerDelegate.java | 47 +++ .../hbase/fs/legacy/cleaner/HFileCleaner.java | 72 ++++ .../hbase/fs/legacy/cleaner/HFileLinkCleaner.java | 113 ++++++ .../hadoop/hbase/fs/legacy/cleaner/LogCleaner.java | 56 +++ .../fs/legacy/cleaner/TimeToLiveHFileCleaner.java | 64 ++++ .../fs/legacy/cleaner/TimeToLiveLogCleaner.java | 73 ++++ .../fs/legacy/snapshot/SnapshotFileCache.java | 390 ++++++++++++++++++++ .../fs/legacy/snapshot/SnapshotHFileCleaner.java | 126 +++++++ .../java/org/apache/hadoop/hbase/io/FileLink.java | 2 +- .../hadoop/hbase/master/AssignmentManager.java | 44 +-- .../org/apache/hadoop/hbase/master/HMaster.java | 39 +- .../hadoop/hbase/master/MasterStatusServlet.java | 3 +- .../master/cleaner/BaseFileCleanerDelegate.java | 56 --- .../master/cleaner/BaseHFileCleanerDelegate.java | 54 --- .../master/cleaner/BaseLogCleanerDelegate.java | 57 --- .../hadoop/hbase/master/cleaner/CleanerChore.java | 294 --------------- .../hbase/master/cleaner/FileCleanerDelegate.java | 47 --- .../hadoop/hbase/master/cleaner/HFileCleaner.java | 72 ---- .../hbase/master/cleaner/HFileLinkCleaner.java | 113 ------ .../hadoop/hbase/master/cleaner/LogCleaner.java | 56 --- .../master/cleaner/TimeToLiveHFileCleaner.java | 64 ---- .../hbase/master/cleaner/TimeToLiveLogCleaner.java | 73 ---- .../hbase/master/snapshot/SnapshotFileCache.java | 389 -------------------- .../master/snapshot/SnapshotHFileCleaner.java | 126 ------- .../hbase/master/snapshot/SnapshotManager.java | 28 +- .../hadoop/hbase/protobuf/ServerProtobufUtil.java | 122 +------ .../regionserver/StorefileRefresherChore.java | 2 +- .../master/ReplicationHFileCleaner.java | 5 +- .../replication/master/ReplicationLogCleaner.java | 6 +- .../replication/regionserver/Replication.java | 2 +- .../java/org/apache/hadoop/hbase/util/FSUtils.java | 57 --- .../main/resources/hbase-webapps/master/table.jsp | 4 +- .../apache/hadoop/hbase/HBaseTestingUtility.java | 14 + .../hadoop/hbase/backup/TestHFileArchiving.java | 17 +- .../example/TestZooKeeperTableArchiveClient.java | 4 +- .../hbase/client/TestCloneSnapshotFromClient.java | 6 +- .../client/TestRestoreSnapshotFromClient.java | 6 +- .../hbase/fs/legacy/cleaner/TestCleanerChore.java | 319 ++++++++++++++++ .../hbase/fs/legacy/cleaner/TestHFileCleaner.java | 263 ++++++++++++++ .../fs/legacy/cleaner/TestHFileLinkCleaner.java | 201 ++++++++++ .../hbase/fs/legacy/cleaner/TestLogsCleaner.java | 308 ++++++++++++++++ .../fs/legacy/cleaner/TestSnapshotFromMaster.java | 394 ++++++++++++++++++++ .../fs/legacy/snapshot/TestSnapshotFileCache.java | 288 +++++++++++++++ .../legacy/snapshot/TestSnapshotHFileCleaner.java | 190 ++++++++++ .../hbase/master/cleaner/TestCleanerChore.java | 319 ---------------- .../hbase/master/cleaner/TestHFileCleaner.java | 263 -------------- .../hbase/master/cleaner/TestHFileLinkCleaner.java | 201 ---------- .../hbase/master/cleaner/TestLogsCleaner.java | 309 ---------------- .../cleaner/TestReplicationHFileCleaner.java | 341 ----------------- .../master/cleaner/TestSnapshotFromMaster.java | 404 --------------------- .../master/snapshot/TestSnapshotFileCache.java | 282 -------------- .../master/snapshot/TestSnapshotHFileCleaner.java | 190 ---------- .../hbase/master/snapshot/TestSnapshotManager.java | 5 +- .../master/TestReplicationHFileCleaner.java | 341 +++++++++++++++++ src/main/asciidoc/_chapters/hbase-default.adoc | 4 +- 64 files changed, 3997 insertions(+), 4030 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseFileCleanerDelegate.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseHFileCleanerDelegate.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseLogCleanerDelegate.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/CleanerChore.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/FileCleanerDelegate.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileCleaner.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileLinkCleaner.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/LogCleaner.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TimeToLiveHFileCleaner.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TimeToLiveLogCleaner.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotFileCache.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotHFileCleaner.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestCleanerChore.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileCleaner.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileLinkCleaner.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestLogsCleaner.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestSnapshotFromMaster.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotFileCache.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotHFileCleaner.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationHFileCleaner.java diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 4f769cb..8bb46f5 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -122,13 +122,13 @@ possible configurations would overwhelm and obscure the important. hbase.master.logcleaner.plugins - org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner - A comma-separated list of BaseLogCleanerDelegate invoked by - the LogsCleaner service. These WAL cleaners are called in order, - so put the cleaner that prunes the most files in front. To - implement your own BaseLogCleanerDelegate, just put it in HBase's classpath - and add the fully qualified class name here. Always add the above - default log cleaners in the list. + org.apache.hadoop.hbase.fs.legacy.cleaner.TimeToLiveLogCleaner + This property is used when 'hbase.storage.type' is set to 'legacy'. + A comma-separated list of BaseLogCleanerDelegate invoked by the LogsCleaner + service. These WAL cleaners are called in order, so put the cleaner that prunes + the most files in front. To implement your own BaseLogCleanerDelegate, just put + it in HBase's classpath and add the fully qualified class name here. Always add + the above default log cleaners in the list. hbase.master.logcleaner.ttl @@ -138,14 +138,14 @@ possible configurations would overwhelm and obscure the important. hbase.master.hfilecleaner.plugins - org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner - A comma-separated list of BaseHFileCleanerDelegate invoked by - the HFileCleaner service. These HFiles cleaners are called in order, - so put the cleaner that prunes the most files in front. To - implement your own BaseHFileCleanerDelegate, just put it in HBase's classpath - and add the fully qualified class name here. Always add the above - default log cleaners in the list as they will be overwritten in - hbase-site.xml. + org.apache.hadoop.hbase.fs.legacy.cleaner.TimeToLiveHFileCleaner + This property is used when 'hbase.storage.type' is set to 'legacy'. + A comma-separated list of BaseHFileCleanerDelegate invoked by the HFileCleaner + service. These HFiles cleaners are called in order, so put the cleaner that + prunes the most files in front. To implement your own BaseHFileCleanerDelegate, + just put it in HBase's classpath and add the fully qualified class name here. + Always add the above default log cleaners in the list as they will be + overwritten in hbase-site.xml. hbase.master.infoserver.redirect diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/LongTermArchivingHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/LongTermArchivingHFileCleaner.java index 09a6659..ad2e8b4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/LongTermArchivingHFileCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/LongTermArchivingHFileCleaner.java @@ -27,7 +27,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate; +import org.apache.hadoop.hbase.fs.legacy.cleaner.BaseHFileCleanerDelegate; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.zookeeper.KeeperException; @@ -36,7 +36,7 @@ import org.apache.zookeeper.KeeperException; * currently being archived. *

* This only works properly if the - * {@link org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner} + * {@link org.apache.hadoop.hbase.fs.legacy.cleaner.TimeToLiveHFileCleaner} * is also enabled (it always should be), since it may take a little time * for the ZK notification to propagate, in which case we may accidentally * delete some files. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java index 0ccae4a..2f3b4a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/MasterStorage.java @@ -22,18 +22,22 @@ package org.apache.hadoop.hbase.fs; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.ClusterId; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.ScheduledChore; +import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.fs.legacy.LegacyMasterStorage; import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor; import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier; @@ -62,6 +66,39 @@ public abstract class MasterStorage { public FileSystem getFileSystem() { return fs; } // TODO: definitely remove public IDENTIFIER getRootContainer() { return rootContainer; } + /** + * Get Chores that are required to be run from time to time for the underlying MasterStorage + * implementation. A few setup methods e.g. {@link #enableSnapshots()} may have their own chores. + * The returned list of chores or their configuration may vary depending on when in sequence + * this method is called with respect to other methods. Generally, a call to this method for + * getting and scheduling chores, needs to be after storage is setup properly by calling those + * methods first. + * + * Please refer to the documentation of specific method implementation for more details. + * + * @param stopper the stopper + * @return storage chores. + */ + public Iterable getChores(Stoppable stopper, Map params) { + return new ArrayList<>(); + } + + /** + * This method should be called to prepare storage implementation/s for snapshots. The default + * implementation does nothing. MasterStorage subclasses need to override this method to + * provide specific preparatory steps. + */ + public void enableSnapshots() { + return; + } + + /** + * Returns true if MasterStorage is prepared for snapshots + */ + public boolean isSnapshotsEnabled() { + return true; + } + // ========================================================================== // PUBLIC Interfaces - Visitors // ========================================================================== @@ -178,6 +215,45 @@ public abstract class MasterStorage { */ public abstract void archiveTable(StorageContext ctx, TableName tableName) throws IOException; + /** + * Runs through all tables and checks how many stores for each table + * have more than one file in them. Checks -ROOT- and hbase:meta too. The total + * percentage across all tables is stored under the special key "-TOTAL-". + * + * @return A map for each table and its percentage. + * + * @throws IOException When scanning the directory fails. + */ + public Map getTableFragmentation() throws IOException { + final Map frags = new HashMap<>(); + int cfCountTotal = 0; + int cfFragTotal = 0; + + for (TableName table: getTables()) { + int cfCount = 0; + int cfFrag = 0; + for (HRegionInfo hri: getRegions(table)) { + RegionStorage rs = getRegionStorage(hri); + final Collection families = rs.getFamilies(); + for (String family: families) { + cfCount++; + cfCountTotal++; + if (rs.getStoreFiles(family).size() > 1) { + cfFrag++; + cfFragTotal++; + } + } + } + // compute percentage per table and store in result list + frags.put(table.getNameAsString(), + cfCount == 0? 0: Math.round((float) cfFrag / cfCount * 100)); + } + // set overall percentage for all tables + frags.put("-TOTAL-", + cfCountTotal == 0? 0: Math.round((float) cfFragTotal / cfCountTotal * 100)); + return frags; + } + // ========================================================================== // PUBLIC Methods - Table Region related // ========================================================================== @@ -232,6 +308,16 @@ public abstract class MasterStorage { public abstract void archiveRegion(HRegionInfo regionInfo) throws IOException; // ========================================================================== + // PUBLIC Methods - WAL + // ========================================================================== + + /** + * Returns true if given region server has non-empty WAL files + * @param serverName + */ + public abstract boolean hasWALs(String serverName) throws IOException; + + // ========================================================================== // PUBLIC Methods - visitors // ========================================================================== public void visitStoreFiles(StoreFileVisitor visitor) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java index ce29bb2..2906f91 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyLayout.java @@ -124,4 +124,8 @@ public final class LegacyLayout { public static Path getBulkDir(Path rootDir) { return new Path(rootDir, HConstants.BULKLOAD_STAGING_DIR_NAME); } + + public static Path getOldLogDir(final Path rootDir) { + return new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterStorage.java index b6443de..aa4de2c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterStorage.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/LegacyMasterStorage.java @@ -24,6 +24,9 @@ import java.io.FileNotFoundException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,8 +37,19 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.ClusterId; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.ScheduledChore; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileLinkCleaner; +import org.apache.hadoop.hbase.fs.legacy.cleaner.LogCleaner; +import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotHFileCleaner; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -80,6 +94,8 @@ public class LegacyMasterStorage extends MasterStorage { private final boolean isSecurityEnabled; + public static final String SPLITTING_EXT = "-splitting"; + public LegacyMasterStorage(Configuration conf, FileSystem fs, LegacyPathIdentifier rootDir) { super(conf, fs, rootDir); @@ -97,6 +113,54 @@ public class LegacyMasterStorage extends MasterStorage { this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")); } + @Override + public Iterable getChores(Stoppable stopper, Map params) { + ArrayList chores = (ArrayList) super.getChores(stopper, params); + + int cleanerInterval = getConfiguration().getInt("hbase.master.cleaner.interval", 60 * 1000); + // add log cleaner chore + chores.add(new LogCleaner(cleanerInterval, stopper, getConfiguration(), getFileSystem(), + LegacyLayout.getOldLogDir(getRootContainer().path))); + // add hfile archive cleaner chore + chores.add(new HFileCleaner(cleanerInterval, stopper, getConfiguration(), getFileSystem(), + LegacyLayout.getArchiveDir(getRootContainer().path), params)); + + return chores; + } + + /** + * This method modifies chores configuration for snapshots. Please call this method before + * instantiating and scheduling list of chores with {@link #getChores(Stoppable, Map)}. + */ + @Override + public void enableSnapshots() { + super.enableSnapshots(); + if (!isSnapshotsEnabled()) { + // Extract cleaners from conf + Set hfileCleaners = new HashSet<>(); + String[] cleaners = getConfiguration().getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS); + if (cleaners != null) Collections.addAll(hfileCleaners, cleaners); + + // add snapshot related cleaners + hfileCleaners.add(SnapshotHFileCleaner.class.getName()); + hfileCleaners.add(HFileLinkCleaner.class.getName()); + + // Set cleaners conf + getConfiguration().setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, + hfileCleaners.toArray(new String[hfileCleaners.size()])); + } + } + + @Override + public boolean isSnapshotsEnabled() { + // Extract cleaners from conf + Set hfileCleaners = new HashSet<>(); + String[] cleaners = getConfiguration().getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS); + if (cleaners != null) Collections.addAll(hfileCleaners, cleaners); + return hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) && + hfileCleaners.contains(HFileLinkCleaner.class.getName()); + } + // ========================================================================== // PUBLIC Methods - Namespace related // ========================================================================== @@ -268,6 +332,53 @@ public class LegacyMasterStorage extends MasterStorage { } // ========================================================================== + // PUBLIC - WAL + // ========================================================================== + @Override + public boolean hasWALs(String serverName) throws IOException { + Path logDir = new Path(getRootContainer().path, new StringBuilder( + HConstants.HREGION_LOGDIR_NAME).append("/").append(serverName).toString()); + Path splitDir = logDir.suffix(SPLITTING_EXT); + + return checkWALs(logDir) || checkWALs(splitDir); + } + + + // ========================================================================== + // PRIVATE - WAL + // ========================================================================== + + private boolean checkWALs(Path dir) throws IOException { + FileSystem fs = getFileSystem(); + + if (!fs.exists(dir)) { + LOG.debug(dir + " not found!"); + return false; + } else if (!fs.getFileStatus(dir).isDirectory()) { + LOG.warn(dir + " is not a directory"); + return false; + } + + FileStatus[] files = FSUtils.listStatus(fs, dir); + if (files == null || files.length == 0) { + LOG.debug(dir + " has no files"); + return false; + } + + for (FileStatus dentry: files) { + if (dentry.isFile() && dentry.getLen() > 0) { + LOG.debug(dir + " has a non-empty file: " + dentry.getPath()); + return true; + } else if (dentry.isDirectory() && checkWALs(dentry.getPath())) { + LOG.debug(dentry + " is a directory and has a non-empty file!"); + return true; + } + } + LOG.debug("Found zero non-empty wal files for: " + dir); + return false; + } + + // ========================================================================== // PROTECTED Methods - Bootstrap // ========================================================================== diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseFileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseFileCleanerDelegate.java new file mode 100644 index 0000000..6ab33a7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseFileCleanerDelegate.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hbase.BaseConfigurable; + +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; + +import java.util.Map; + +/** + * Base class for file cleaners which allows subclasses to implement a simple + * isFileDeletable method (which used to be the FileCleanerDelegate contract). + */ +public abstract class BaseFileCleanerDelegate extends BaseConfigurable +implements FileCleanerDelegate { + + @Override + public Iterable getDeletableFiles(Iterable files) { + return Iterables.filter(files, new Predicate() { + @Override + public boolean apply(FileStatus file) { + return isFileDeletable(file); + }}); + } + + @Override + public void init(Map params) { + // subclass could override it if needed. + } + + /** + * Should the master delete the file or keep it? + * @param fStat file status of the file to check + * @return true if the file is deletable, false if not + */ + protected abstract boolean isFileDeletable(FileStatus fStat); + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseHFileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseHFileCleanerDelegate.java new file mode 100644 index 0000000..a5018bd --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseHFileCleanerDelegate.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * Base class for the hfile cleaning function inside the master. By default, only the + * {@link TimeToLiveHFileCleaner} is called. + *

+ * If other effects are needed, implement your own LogCleanerDelegate and add it to the + * configuration "hbase.master.hfilecleaner.plugins", which is a comma-separated list of fully + * qualified class names. The HFileCleaner will build the cleaner chain in + * order the order specified by the configuration. + *

+ *

+ * For subclasses, setConf will be called exactly once before using the cleaner. + *

+ *

+ * Since {@link BaseHFileCleanerDelegate HFileCleanerDelegates} are created in + * HFileCleaner by reflection, classes that implements this interface must + * provide a default constructor. + *

+ */ +@InterfaceAudience.Private +public abstract class BaseHFileCleanerDelegate extends BaseFileCleanerDelegate { + + private boolean stopped = false; + + @Override + public void stop(String why) { + this.stopped = true; + } + + @Override + public boolean isStopped() { + return this.stopped; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseLogCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseLogCleanerDelegate.java new file mode 100644 index 0000000..4d4e31f --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/BaseLogCleanerDelegate.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.fs.FileStatus; + +/** + * Base class for the log cleaning function inside the master. By default, two + * cleaners: TimeToLiveLogCleaner and + * ReplicationLogCleaner are called in order. So if other effects + * are needed, implement your own LogCleanerDelegate and add it to the + * configuration "hbase.master.logcleaner.plugins", which is a comma-separated + * list of fully qualified class names. LogsCleaner will add it to the chain. + *

+ * HBase ships with LogsCleaner as the default implementation. + *

+ * This interface extends Configurable, so setConf needs to be called once + * before using the cleaner. Since LogCleanerDelegates are created in + * LogsCleaner by reflection. Classes that implements this interface should + * provide a default constructor. + */ +@InterfaceAudience.Private +public abstract class BaseLogCleanerDelegate extends BaseFileCleanerDelegate { + + @Override + public boolean isFileDeletable(FileStatus fStat) { + return isLogDeletable(fStat); + } + + /** + * Should the master delete the log or keep it? + *

+ * Implementing classes should override {@link #isFileDeletable(FileStatus)} instead. + * @param fStat file status of the file + * @return true if the log is deletable, false (default) if not + */ + @Deprecated + public boolean isLogDeletable(FileStatus fStat) { + return false; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/CleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/CleanerChore.java new file mode 100644 index 0000000..300c6f8 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/CleanerChore.java @@ -0,0 +1,294 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ScheduledChore; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.ipc.RemoteException; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Abstract Cleaner that uses a chain of delegates to clean a directory of files + * @param Cleaner delegate class that is dynamically loaded from configuration + */ +public abstract class CleanerChore extends ScheduledChore { + + private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName()); + + private final FileSystem fs; + private final Path oldFileDir; + private final Configuration conf; + protected List cleanersChain; + protected Map params; + + public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf, + FileSystem fs, Path oldFileDir, String confKey) { + this(name, sleepPeriod, s, conf, fs, oldFileDir, confKey, null); + } + + /** + * @param name name of the chore being run + * @param sleepPeriod the period of time to sleep between each run + * @param s the stopper + * @param conf configuration to use + * @param fs handle to the FS + * @param oldFileDir the path to the archived files + * @param confKey configuration key for the classes to instantiate + * @param params members could be used in cleaner + */ + public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf, + FileSystem fs, Path oldFileDir, String confKey, Map params) { + super(name, s, sleepPeriod); + this.fs = fs; + this.oldFileDir = oldFileDir; + this.conf = conf; + this.params = params; + initCleanerChain(confKey); + } + + + /** + * Validate the file to see if it even belongs in the directory. If it is valid, then the file + * will go through the cleaner delegates, but otherwise the file is just deleted. + * @param file full {@link Path} of the file to be checked + * @return true if the file is valid, false otherwise + */ + protected abstract boolean validate(Path file); + + /** + * Instantiate and initialize all the file cleaners set in the configuration + * @param confKey key to get the file cleaner classes from the configuration + */ + private void initCleanerChain(String confKey) { + this.cleanersChain = new LinkedList(); + String[] logCleaners = conf.getStrings(confKey); + if (logCleaners != null) { + for (String className : logCleaners) { + T logCleaner = newFileCleaner(className, conf); + if (logCleaner != null) { + LOG.debug("initialize cleaner=" + className); + this.cleanersChain.add(logCleaner); + } + } + } + } + + /** + * A utility method to create new instances of LogCleanerDelegate based on the class name of the + * LogCleanerDelegate. + * @param className fully qualified class name of the LogCleanerDelegate + * @param conf + * @return the new instance + */ + private T newFileCleaner(String className, Configuration conf) { + try { + Class c = Class.forName(className).asSubclass( + FileCleanerDelegate.class); + @SuppressWarnings("unchecked") + T cleaner = (T) c.newInstance(); + cleaner.setConf(conf); + cleaner.init(this.params); + return cleaner; + } catch (Exception e) { + LOG.warn("Can NOT create CleanerDelegate: " + className, e); + // skipping if can't instantiate + return null; + } + } + + @Override + protected void chore() { + try { + FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir); + checkAndDeleteEntries(files); + } catch (IOException e) { + e = e instanceof RemoteException ? + ((RemoteException)e).unwrapRemoteException() : e; + LOG.warn("Error while cleaning the logs", e); + } + } + + /** + * Loop over the given directory entries, and check whether they can be deleted. + * If an entry is itself a directory it will be recursively checked and deleted itself iff + * all subentries are deleted (and no new subentries are added in the mean time) + * + * @param entries directory entries to check + * @return true if all entries were successfully deleted + */ + private boolean checkAndDeleteEntries(FileStatus[] entries) { + if (entries == null) { + return true; + } + boolean allEntriesDeleted = true; + List files = Lists.newArrayListWithCapacity(entries.length); + for (FileStatus child : entries) { + Path path = child.getPath(); + if (child.isDirectory()) { + // for each subdirectory delete it and all entries if possible + if (!checkAndDeleteDirectory(path)) { + allEntriesDeleted = false; + } + } else { + // collect all files to attempt to delete in one batch + files.add(child); + } + } + if (!checkAndDeleteFiles(files)) { + allEntriesDeleted = false; + } + return allEntriesDeleted; + } + + /** + * Attempt to delete a directory and all files under that directory. Each child file is passed + * through the delegates to see if it can be deleted. If the directory has no children when the + * cleaners have finished it is deleted. + *

+ * If new children files are added between checks of the directory, the directory will not + * be deleted. + * @param dir directory to check + * @return true if the directory was deleted, false otherwise. + */ + @VisibleForTesting boolean checkAndDeleteDirectory(Path dir) { + if (LOG.isTraceEnabled()) { + LOG.trace("Checking directory: " + dir); + } + + try { + FileStatus[] children = FSUtils.listStatus(fs, dir); + boolean allChildrenDeleted = checkAndDeleteEntries(children); + + // if the directory still has children, we can't delete it, so we are done + if (!allChildrenDeleted) return false; + } catch (IOException e) { + e = e instanceof RemoteException ? + ((RemoteException)e).unwrapRemoteException() : e; + LOG.warn("Error while listing directory: " + dir, e); + // couldn't list directory, so don't try to delete, and don't return success + return false; + } + + // otherwise, all the children (that we know about) have been deleted, so we should try to + // delete this directory. However, don't do so recursively so we don't delete files that have + // been added since we last checked. + try { + return fs.delete(dir, false); + } catch (IOException e) { + if (LOG.isTraceEnabled()) { + LOG.trace("Couldn't delete directory: " + dir, e); + } + // couldn't delete w/o exception, so we can't return success. + return false; + } + } + + /** + * Run the given files through each of the cleaners to see if it should be deleted, deleting it if + * necessary. + * @param files List of FileStatus for the files to check (and possibly delete) + * @return true iff successfully deleted all files + */ + private boolean checkAndDeleteFiles(List files) { + // first check to see if the path is valid + List validFiles = Lists.newArrayListWithCapacity(files.size()); + List invalidFiles = Lists.newArrayList(); + for (FileStatus file : files) { + if (validate(file.getPath())) { + validFiles.add(file); + } else { + LOG.warn("Found a wrongly formatted file: " + file.getPath() + " - will delete it."); + invalidFiles.add(file); + } + } + + Iterable deletableValidFiles = validFiles; + // check each of the cleaners for the valid files + for (T cleaner : cleanersChain) { + if (cleaner.isStopped() || this.getStopper().isStopped()) { + LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any more files in:" + + this.oldFileDir); + return false; + } + + Iterable filteredFiles = cleaner.getDeletableFiles(deletableValidFiles); + + // trace which cleaner is holding on to each file + if (LOG.isTraceEnabled()) { + ImmutableSet filteredFileSet = ImmutableSet.copyOf(filteredFiles); + for (FileStatus file : deletableValidFiles) { + if (!filteredFileSet.contains(file)) { + LOG.trace(file.getPath() + " is not deletable according to:" + cleaner); + } + } + } + + deletableValidFiles = filteredFiles; + } + + Iterable filesToDelete = Iterables.concat(invalidFiles, deletableValidFiles); + int deletedFileCount = 0; + for (FileStatus file : filesToDelete) { + Path filePath = file.getPath(); + if (LOG.isDebugEnabled()) { + LOG.debug("Removing: " + filePath + " from archive"); + } + try { + boolean success = this.fs.delete(filePath, false); + if (success) { + deletedFileCount++; + } else { + LOG.warn("Attempted to delete:" + filePath + + ", but couldn't. Run cleaner chain and attempt to delete on next pass."); + } + } catch (IOException e) { + e = e instanceof RemoteException ? + ((RemoteException)e).unwrapRemoteException() : e; + LOG.warn("Error while deleting: " + filePath, e); + } + } + + return deletedFileCount == files.size(); + } + + @Override + public void cleanup() { + for (T lc : this.cleanersChain) { + try { + lc.stop("Exiting"); + } catch (Throwable t) { + LOG.warn("Stopping", t); + } + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/FileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/FileCleanerDelegate.java new file mode 100644 index 0000000..0291640 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/FileCleanerDelegate.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hbase.Stoppable; + +import java.util.Map; + +/** + * General interface for cleaning files from a folder (generally an archive or + * backup folder). These are chained via the {@link CleanerChore} to determine + * if a given file should be deleted. + */ +@InterfaceAudience.Private +public interface FileCleanerDelegate extends Configurable, Stoppable { + + /** + * Determines which of the given files are safe to delete + * @param files files to check for deletion + * @return files that are ok to delete according to this cleaner + */ + Iterable getDeletableFiles(Iterable files); + + + /** + * this method is used to pass some instance into subclass + * */ + void init(Map params); +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileCleaner.java new file mode 100644 index 0000000..7137432 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileCleaner.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +/** + * This Chore, every time it runs, will clear the HFiles in the hfile archive + * folder that are deletable for each HFile cleaner in the chain. + */ +@InterfaceAudience.Private +public class HFileCleaner extends CleanerChore { + + public static final String MASTER_HFILE_CLEANER_PLUGINS = "hbase.master.hfilecleaner.plugins"; + + public HFileCleaner(final int period, final Stoppable stopper, Configuration conf, FileSystem fs, + Path directory) { + this(period, stopper, conf, fs, directory, null); + } + + /** + * @param period the period of time to sleep between each run + * @param stopper the stopper + * @param conf configuration to use + * @param fs handle to the FS + * @param directory directory to be cleaned + * @param params params could be used in subclass of BaseHFileCleanerDelegate + */ + public HFileCleaner(final int period, final Stoppable stopper, Configuration conf, FileSystem fs, + Path directory, Map params) { + super("HFileCleaner", period, stopper, conf, fs, + directory, MASTER_HFILE_CLEANER_PLUGINS, params); + } + + @Override + protected boolean validate(Path file) { + if (HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent())) { + return true; + } + return StoreFileInfo.validateStoreFileName(file.getName()); + } + + /** + * Exposed for TESTING! + */ + public List getDelegatesForTesting() { + return this.cleanersChain; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileLinkCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileLinkCleaner.java new file mode 100644 index 0000000..7c1f4d2 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileLinkCleaner.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.util.FSUtils; + +/** + * HFileLink cleaner that determines if a hfile should be deleted. + * HFiles can be deleted only if there're no links to them. + * + * When a HFileLink is created a back reference file is created in: + * /hbase/archive/table/region/cf/.links-hfile/ref-region.ref-table + * To check if the hfile can be deleted the back references folder must be empty. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) +public class HFileLinkCleaner extends BaseHFileCleanerDelegate { + private static final Log LOG = LogFactory.getLog(HFileLinkCleaner.class); + + private FileSystem fs = null; + + @Override + public synchronized boolean isFileDeletable(FileStatus fStat) { + if (this.fs == null) return false; + Path filePath = fStat.getPath(); + // HFile Link is always deletable + if (HFileLink.isHFileLink(filePath)) return true; + + // If the file is inside a link references directory, means that it is a back ref link. + // The back ref can be deleted only if the referenced file doesn't exists. + Path parentDir = filePath.getParent(); + if (HFileLink.isBackReferencesDir(parentDir)) { + Path hfilePath = null; + try { + // Also check if the HFile is in the HBASE_TEMP_DIRECTORY; this is where the referenced + // file gets created when cloning a snapshot. + hfilePath = HFileLink.getHFileFromBackReference( + new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY), filePath); + if (fs.exists(hfilePath)) { + return false; + } + // check whether the HFileLink still exists in mob dir. + hfilePath = HFileLink.getHFileFromBackReference(MobUtils.getMobHome(getConf()), filePath); + if (fs.exists(hfilePath)) { + return false; + } + hfilePath = HFileLink.getHFileFromBackReference(FSUtils.getRootDir(getConf()), filePath); + return !fs.exists(hfilePath); + } catch (IOException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Couldn't verify if the referenced file still exists, keep it just in case: " + + hfilePath); + } + return false; + } + } + + // HFile is deletable only if has no links + Path backRefDir = null; + try { + backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName()); + return FSUtils.listStatus(fs, backRefDir) == null; + } catch (IOException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Couldn't get the references, not deleting file, just in case. filePath=" + + filePath + ", backRefDir=" + backRefDir); + } + return false; + } + } + + @Override + public synchronized void setConf(Configuration conf) { + super.setConf(conf); + + // setup filesystem + try { + this.fs = FileSystem.get(this.getConf()); + } catch (IOException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Couldn't instantiate the file system, not deleting file, just in case. " + + FileSystem.FS_DEFAULT_NAME_KEY + "=" + + getConf().get(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS)); + } + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/LogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/LogCleaner.java new file mode 100644 index 0000000..430c482 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/LogCleaner.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; + +/** + * This Chore, every time it runs, will attempt to delete the WALs in the old logs folder. The WAL + * is only deleted if none of the cleaner delegates says otherwise. + * @see BaseLogCleanerDelegate + */ +@InterfaceAudience.Private +public class LogCleaner extends CleanerChore { + private static final Log LOG = LogFactory.getLog(LogCleaner.class.getName()); + + /** + * @param p the period of time to sleep between each run + * @param s the stopper + * @param conf configuration to use + * @param fs handle to the FS + * @param oldLogDir the path to the archived logs + */ + public LogCleaner(final int p, final Stoppable s, Configuration conf, FileSystem fs, + Path oldLogDir) { + super("LogsCleaner", p, s, conf, fs, oldLogDir, HBASE_MASTER_LOGCLEANER_PLUGINS); + } + + @Override + protected boolean validate(Path file) { + return AbstractFSWALProvider.validateWALFilename(file.getName()); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TimeToLiveHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TimeToLiveHFileCleaner.java new file mode 100644 index 0000000..177df38 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TimeToLiveHFileCleaner.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +/** + * HFile cleaner that uses the timestamp of the hfile to determine if it should be deleted. By + * default they are allowed to live for {@value #DEFAULT_TTL} + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) +public class TimeToLiveHFileCleaner extends BaseHFileCleanerDelegate { + + private static final Log LOG = LogFactory.getLog(TimeToLiveHFileCleaner.class.getName()); + public static final String TTL_CONF_KEY = "hbase.master.hfilecleaner.ttl"; + // default ttl = 5 minutes + public static final long DEFAULT_TTL = 60000 * 5; + // Configured time a hfile can be kept after it was moved to the archive + private long ttl; + + @Override + public void setConf(Configuration conf) { + this.ttl = conf.getLong(TTL_CONF_KEY, DEFAULT_TTL); + super.setConf(conf); + } + + @Override + public boolean isFileDeletable(FileStatus fStat) { + long currentTime = EnvironmentEdgeManager.currentTime(); + long time = fStat.getModificationTime(); + long life = currentTime - time; + if (LOG.isTraceEnabled()) { + LOG.trace("HFile life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: " + + time); + } + if (life < 0) { + LOG.warn("Found a hfile (" + fStat.getPath() + ") newer than current time (" + currentTime + + " < " + time + "), probably a clock skew"); + return false; + } + return life > ttl; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TimeToLiveLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TimeToLiveLogCleaner.java new file mode 100644 index 0000000..e3719ba --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TimeToLiveLogCleaner.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +/** + * Log cleaner that uses the timestamp of the wal to determine if it should + * be deleted. By default they are allowed to live for 10 minutes. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) +public class TimeToLiveLogCleaner extends BaseLogCleanerDelegate { + private static final Log LOG = LogFactory.getLog(TimeToLiveLogCleaner.class.getName()); + // Configured time a log can be kept after it was closed + private long ttl; + private boolean stopped = false; + + @Override + public boolean isLogDeletable(FileStatus fStat) { + long currentTime = EnvironmentEdgeManager.currentTime(); + long time = fStat.getModificationTime(); + long life = currentTime - time; + + if (LOG.isTraceEnabled()) { + LOG.trace("Log life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: " + + time); + } + if (life < 0) { + LOG.warn("Found a log (" + fStat.getPath() + ") newer than current time (" + currentTime + + " < " + time + "), probably a clock skew"); + return false; + } + return life > ttl; + } + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + this.ttl = conf.getLong("hbase.master.logcleaner.ttl", 600000); + } + + + @Override + public void stop(String why) { + this.stopped = true; + } + + @Override + public boolean isStopped() { + return this.stopped; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotFileCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotFileCache.java new file mode 100644 index 0000000..f9f1c67 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotFileCache.java @@ -0,0 +1,390 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.locks.ReentrantLock; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.util.FSUtils; + +/** + * Intelligently keep track of all the files for all the snapshots. + *

+ * A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache + * miss the directory modification time is used to ensure that we don't rescan directories that we + * already have in cache. We only check the modification times of the snapshot directories + * (/hbase/.snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache. + *

+ * New snapshots will be added to the cache and deleted snapshots will be removed when we refresh + * the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself, + * we will ignore updates to that snapshot's files. + *

+ * This is sufficient because each snapshot has its own directory and is added via an atomic rename + * once, when the snapshot is created. We don't need to worry about the data in the snapshot + * being run. + *

+ * Further, the cache is periodically refreshed ensure that files in snapshots that were deleted are + * also removed from the cache. + *

+ * A {@link SnapshotFileCache.SnapshotFileInspector} must be passed when creating this to + * allow extraction of files under /hbase/.snapshot/[snapshot name] directory, for each snapshot. + * This allows you to only cache files under, for instance, all the logs in the .logs directory or + * all the files under all the regions. + *

+ * this also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid + * snapshots and will attempt to cache files from those snapshots as well. + *

+ * Queries about a given file are thread-safe with respect to multiple queries and cache refreshes. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class SnapshotFileCache implements Stoppable { + interface SnapshotFileInspector { + /** + * Returns a collection of file names needed by the snapshot. + * @param snapshotDir {@link Path} to the snapshot directory to scan. + * @return the collection of file names needed by the snapshot. + */ + Collection filesUnderSnapshot(final Path snapshotDir) throws IOException; + } + + private static final Log LOG = LogFactory.getLog(SnapshotFileCache.class); + private volatile boolean stop = false; + private final FileSystem fs; + private final SnapshotFileInspector fileInspector; + private final Path snapshotDir; + private final Set cache = new HashSet(); + /** + * This is a helper map of information about the snapshot directories so we don't need to rescan + * them if they haven't changed since the last time we looked. + */ + private final Map snapshots = + new HashMap(); + private final Timer refreshTimer; + + private long lastModifiedTime = Long.MIN_VALUE; + + /** + * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the + * filesystem. + *

+ * Immediately loads the file cache. + * @param conf to extract the configured {@link FileSystem} where the snapshots are stored and + * hbase root directory + * @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed + * @param refreshThreadName name of the cache refresh thread + * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files. + * @throws IOException if the {@link FileSystem} or root directory cannot be loaded + */ + public SnapshotFileCache(Configuration conf, long cacheRefreshPeriod, String refreshThreadName, + SnapshotFileInspector inspectSnapshotFiles) throws IOException { + this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), 0, cacheRefreshPeriod, + refreshThreadName, inspectSnapshotFiles); + } + + /** + * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the + * filesystem + * @param fs {@link FileSystem} where the snapshots are stored + * @param rootDir hbase root directory + * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed + * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed + * @param refreshThreadName name of the cache refresh thread + * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files. + */ + public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod, + long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) { + this.fs = fs; + this.fileInspector = inspectSnapshotFiles; + this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); + // periodically refresh the file cache to make sure we aren't superfluously saving files. + this.refreshTimer = new Timer(refreshThreadName, true); + this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay, + cacheRefreshPeriod); + } + + /** + * Trigger a cache refresh, even if its before the next cache refresh. Does not affect pending + * cache refreshes. + *

+ * Blocks until the cache is refreshed. + *

+ * Exposed for TESTING. + */ + public void triggerCacheRefreshForTesting() { + try { + SnapshotFileCache.this.refreshCache(); + } catch (IOException e) { + LOG.warn("Failed to refresh snapshot hfile cache!", e); + } + LOG.debug("Current cache:" + cache); + } + + /** + * Check to see if any of the passed file names is contained in any of the snapshots. + * First checks an in-memory cache of the files to keep. If its not in the cache, then the cache + * is refreshed and the cache checked again for that file. + * This ensures that we never return files that exist. + *

+ * Note this may lead to periodic false positives for the file being referenced. Periodically, the + * cache is refreshed even if there are no requests to ensure that the false negatives get removed + * eventually. For instance, suppose you have a file in the snapshot and it gets loaded into the + * cache. Then at some point later that snapshot is deleted. If the cache has not been refreshed + * at that point, cache will still think the file system contains that file and return + * true, even if it is no longer present (false positive). However, if the file never was + * on the filesystem, we will never find it and always return false. + * @param files file to check, NOTE: Relies that files are loaded from hdfs before method + * is called (NOT LAZY) + * @return unReferencedFiles the collection of files that do not have snapshot references + * @throws IOException if there is an unexpected error reaching the filesystem. + */ + // XXX this is inefficient to synchronize on the method, when what we really need to guard against + // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the + // cache, but that seems overkill at the moment and isn't necessarily a bottleneck. + public synchronized Iterable getUnreferencedFiles(Iterable files, + final SnapshotManager snapshotManager) + throws IOException { + List unReferencedFiles = Lists.newArrayList(); + List snapshotsInProgress = null; + boolean refreshed = false; + for (FileStatus file : files) { + String fileName = file.getPath().getName(); + if (!refreshed && !cache.contains(fileName)) { + refreshCache(); + refreshed = true; + } + if (cache.contains(fileName)) { + continue; + } + if (snapshotsInProgress == null) { + snapshotsInProgress = getSnapshotsInProgress(snapshotManager); + } + if (snapshotsInProgress.contains(fileName)) { + continue; + } + unReferencedFiles.add(file); + } + return unReferencedFiles; + } + + private synchronized void refreshCache() throws IOException { + long lastTimestamp = Long.MAX_VALUE; + boolean hasChanges = false; + + // get the status of the snapshots directory and check if it is has changes + try { + FileStatus dirStatus = fs.getFileStatus(snapshotDir); + lastTimestamp = dirStatus.getModificationTime(); + hasChanges |= (lastTimestamp >= lastModifiedTime); + } catch (FileNotFoundException e) { + if (this.cache.size() > 0) { + LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist"); + } + return; + } + + // get the status of the snapshots temporary directory and check if it has changes + // The top-level directory timestamp is not updated, so we have to check the inner-level. + try { + Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME); + FileStatus tempDirStatus = fs.getFileStatus(snapshotTmpDir); + lastTimestamp = Math.min(lastTimestamp, tempDirStatus.getModificationTime()); + hasChanges |= (lastTimestamp >= lastModifiedTime); + if (!hasChanges) { + FileStatus[] tmpSnapshots = FSUtils.listStatus(fs, snapshotDir); + if (tmpSnapshots != null) { + for (FileStatus dirStatus: tmpSnapshots) { + lastTimestamp = Math.min(lastTimestamp, dirStatus.getModificationTime()); + } + hasChanges |= (lastTimestamp >= lastModifiedTime); + } + } + } catch (FileNotFoundException e) { + // Nothing todo, if the tmp dir is empty + } + + // if the snapshot directory wasn't modified since we last check, we are done + if (!hasChanges) { + return; + } + + // directory was modified, so we need to reload our cache + // there could be a slight race here where we miss the cache, check the directory modification + // time, then someone updates the directory, causing us to not scan the directory again. + // However, snapshot directories are only created once, so this isn't an issue. + + // 1. update the modified time + this.lastModifiedTime = lastTimestamp; + + // 2.clear the cache + this.cache.clear(); + Map known = new HashMap(); + + // 3. check each of the snapshot directories + FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir); + if (snapshots == null) { + // remove all the remembered snapshots because we don't have any left + if (LOG.isDebugEnabled() && this.snapshots.size() > 0) { + LOG.debug("No snapshots on-disk, cache empty"); + } + this.snapshots.clear(); + return; + } + + // 3.1 iterate through the on-disk snapshots + for (FileStatus snapshot : snapshots) { + String name = snapshot.getPath().getName(); + // its not the tmp dir, + if (!name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) { + SnapshotDirectoryInfo files = this.snapshots.remove(name); + // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the + // files the latter could occur where I create a snapshot, then delete it, and then make a + // new snapshot with the same name. We will need to update the cache the information from + // that new snapshot, even though it has the same name as the files referenced have + // probably changed. + if (files == null || files.hasBeenModified(snapshot.getModificationTime())) { + // get all files for the snapshot and create a new info + Collection storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath()); + files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles); + } + // 3.2 add all the files to cache + this.cache.addAll(files.getFiles()); + known.put(name, files); + } + } + + // 4. set the snapshots we are tracking + this.snapshots.clear(); + this.snapshots.putAll(known); + } + + @VisibleForTesting List getSnapshotsInProgress( + final SnapshotManager snapshotManager) throws IOException { + List snapshotInProgress = Lists.newArrayList(); + // only add those files to the cache, but not to the known snapshots + Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME); + // only add those files to the cache, but not to the known snapshots + FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir); + if (running != null) { + for (FileStatus run : running) { + ReentrantLock lock = null; + if (snapshotManager != null) { + lock = snapshotManager.getLocks().acquireLock(run.getPath().getName()); + } + try { + snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath())); + } catch (CorruptedSnapshotException e) { + // See HBASE-16464 + if (e.getCause() instanceof FileNotFoundException) { + // If the snapshot is corrupt, we will delete it + fs.delete(run.getPath(), true); + LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause()); + } else { + throw e; + } + } finally { + if (lock != null) { + lock.unlock(); + } + } + } + } + return snapshotInProgress; + } + + /** + * Simple helper task that just periodically attempts to refresh the cache + */ + public class RefreshCacheTask extends TimerTask { + @Override + public void run() { + try { + SnapshotFileCache.this.refreshCache(); + } catch (IOException e) { + LOG.warn("Failed to refresh snapshot hfile cache!", e); + } + } + } + + @Override + public void stop(String why) { + if (!this.stop) { + this.stop = true; + this.refreshTimer.cancel(); + } + + } + + @Override + public boolean isStopped() { + return this.stop; + } + + /** + * Information about a snapshot directory + */ + private static class SnapshotDirectoryInfo { + long lastModified; + Collection files; + + public SnapshotDirectoryInfo(long mtime, Collection files) { + this.lastModified = mtime; + this.files = files; + } + + /** + * @return the hfiles in the snapshot when this was made. + */ + public Collection getFiles() { + return this.files; + } + + /** + * Check if the snapshot directory has been modified + * @param mtime current modification time of the directory + * @return true if it the modification time of the directory is newer time when we + * created this + */ + public boolean hasBeenModified(long mtime) { + return this.lastModified < mtime; + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotHFileCleaner.java new file mode 100644 index 0000000..89704f0 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/snapshot/SnapshotHFileCleaner.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.fs.legacy.cleaner.BaseHFileCleanerDelegate; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; +import org.apache.hadoop.hbase.util.FSUtils; + +/** + * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase + * tables. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) +@InterfaceStability.Evolving +public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate { + private static final Log LOG = LogFactory.getLog(SnapshotHFileCleaner.class); + + /** + * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in + * snapshots (ms) + */ + public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY = + "hbase.master.hfilecleaner.plugins.snapshot.period"; + + /** Refresh cache, by default, every 5 minutes */ + private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000; + + /** File cache for HFiles in the completed and currently running snapshots */ + private SnapshotFileCache cache; + + private MasterServices master; + + @Override + public synchronized Iterable getDeletableFiles(Iterable files) { + try { + if (master != null) return cache.getUnreferencedFiles(files, master.getSnapshotManager()); + } catch (CorruptedSnapshotException cse) { + LOG.debug("Corrupted in-progress snapshot file exception, ignored ", cse); + } catch (IOException e) { + LOG.error("Exception while checking if files were valid, keeping them just in case.", e); + } + return Collections.emptyList(); + } + + @Override + public void init(Map params) { + if (params != null && params.containsKey(HMaster.MASTER)) { + this.master = (MasterServices) params.get(HMaster.MASTER); + } + } + + @Override + protected boolean isFileDeletable(FileStatus fStat) { + return false; + } + + public void setConf(final Configuration conf) { + super.setConf(conf); + try { + long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, + DEFAULT_HFILE_CACHE_REFRESH_PERIOD); + final FileSystem fs = FSUtils.getCurrentFileSystem(conf); + Path rootDir = FSUtils.getRootDir(conf); + cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod, + "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() { + public Collection filesUnderSnapshot(final Path snapshotDir) + throws IOException { + return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir); + } + }); + } catch (IOException e) { + LOG.error("Failed to create cleaner util", e); + } + } + + + @Override + public void stop(String why) { + this.cache.stop(why); + } + + @Override + public boolean isStopped() { + return this.cache.isStopped(); + } + + /** + * Exposed for Testing! + * @return the cache of all hfiles + */ + public SnapshotFileCache getFileCacheForTesting() { + return this.cache; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java index 3caf67f..d3acb05 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java @@ -58,7 +58,7 @@ import org.apache.hadoop.hbase.util.FSUtils; * {@link HFileLink} is a more concrete implementation of the {@code FileLink}. * *

Back-references: - * To help the {@link org.apache.hadoop.hbase.master.cleaner.CleanerChore} to keep track of + * To help the {@link org.apache.hadoop.hbase.fs.legacy.cleaner.CleanerChore} to keep track of * the links to a particular file, during the {@code FileLink} creation, a new file is placed * inside a back-reference directory. There's one back-reference directory for each file that * has links, and in the directory there's one file per link. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index ab058da..7b94cb9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -46,9 +46,6 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CoordinatedStateException; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; @@ -69,6 +66,8 @@ import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.executor.EventHandler; import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.executor.ExecutorService; +import org.apache.hadoop.hbase.fs.MasterStorage; +import org.apache.hadoop.hbase.fs.StorageIdentifier; import org.apache.hadoop.hbase.ipc.FailedServerException; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; @@ -83,13 +82,11 @@ import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.regionserver.RegionServerAbortedException; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.KeyLocker; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.RetryCounter; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.StringUtils; @@ -518,21 +515,17 @@ public class AssignmentManager { // if they don't have any WALs, this restart should be considered as a clean one Set queuedDeadServers = serverManager.getRequeuedDeadServers().keySet(); if (!queuedDeadServers.isEmpty()) { - Configuration conf = server.getConfiguration(); - Path rootdir = FSUtils.getRootDir(conf); - FileSystem fs = rootdir.getFileSystem(conf); + MasterStorage ms = server.getMasterStorage(); for (ServerName serverName: queuedDeadServers) { // In the case of a clean exit, the shutdown handler would have presplit any WALs and // removed empty directories. - Path logDir = new Path(rootdir, - AbstractFSWALProvider.getWALDirectoryName(serverName.toString())); - Path splitDir = logDir.suffix(AbstractFSWALProvider.SPLITTING_EXT); - if (checkWals(fs, logDir) || checkWals(fs, splitDir)) { + if (ms.hasWALs(serverName.toShortString())) { LOG.debug("Found queued dead server " + serverName); failover = true; break; } } + if (!failover) { // We figured that it's not a failover, so no need to // work on these re-queued dead servers any more. @@ -592,33 +585,6 @@ public class AssignmentManager { return failover; } - private boolean checkWals(FileSystem fs, Path dir) throws IOException { - if (!fs.exists(dir)) { - LOG.debug(dir + " doesn't exist"); - return false; - } - if (!fs.getFileStatus(dir).isDirectory()) { - LOG.warn(dir + " is not a directory"); - return false; - } - FileStatus[] files = FSUtils.listStatus(fs, dir); - if (files == null || files.length == 0) { - LOG.debug(dir + " has no files"); - return false; - } - for (int i = 0; i < files.length; i++) { - if (files[i].isFile() && files[i].getLen() > 0) { - LOG.debug(dir + " has a non-empty file: " + files[i].getPath()); - return true; - } else if (files[i].isDirectory() && checkWals(fs, dir)) { - LOG.debug(dir + " is a directory and has a non-empty file: " + files[i].getPath()); - return true; - } - } - LOG.debug("Found 0 non-empty wal files for :" + dir); - return false; - } - /** * When a region is closed, it should be removed from the regionsToReopen * @param hri HRegionInfo of the region which was closed diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 9181579..6b27056 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -99,8 +99,6 @@ import org.apache.hadoop.hbase.master.balancer.BalancerChore; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore; import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; -import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; -import org.apache.hadoop.hbase.master.cleaner.LogCleaner; import org.apache.hadoop.hbase.master.cleaner.ReplicationMetaCleaner; import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; @@ -150,7 +148,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CompressionTest; import org.apache.hadoop.hbase.util.EncryptionTest; import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.HasThread; import org.apache.hadoop.hbase.util.IdLock; import org.apache.hadoop.hbase.util.ModifyRegionUtils; @@ -312,8 +309,7 @@ public class HMaster extends HRegionServer implements MasterServices { CatalogJanitor catalogJanitorChore; private ReplicationMetaCleaner replicationMetaCleaner; - private LogCleaner logCleaner; - private HFileCleaner hfileCleaner; + private Iterable storageManagerChores = new ArrayList<>(); private ExpiredMobFileCleanerChore expiredMobFileCleanerChore; private MobCompactionChore mobCompactChore; private MasterMobCompactionThread mobCompactThread; @@ -961,26 +957,22 @@ public class HMaster extends HRegionServer implements MasterServices { this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1); startProcedureExecutor(); - // Start log cleaner thread - int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000); - this.logCleaner = - new LogCleaner(cleanerInterval, - this, conf, getMasterWalManager().getFileSystem(), - getMasterWalManager().getOldLogDir()); - getChoreService().scheduleChore(logCleaner); - - //start the hfile archive cleaner thread - Path archiveDir = HFileArchiveUtil.getArchivePath(conf); - Map params = new HashMap(); + // Start storage chores + Map params = new HashMap<>(1); params.put(MASTER, this); - this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterStorage() - .getFileSystem(), archiveDir, params); - getChoreService().scheduleChore(hfileCleaner); + storageManagerChores = storageManager.getChores(this, params); + for (ScheduledChore chore: storageManagerChores) { + LOG.info("Starting storage chore: '" + chore.getName() + "'."); + getChoreService().scheduleChore(chore); + } + + // Start log cleaner thread serviceStarted = true; if (LOG.isTraceEnabled()) { LOG.trace("Started service threads"); } + int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000); replicationMetaCleaner = new ReplicationMetaCleaner(this, this, cleanerInterval); getChoreService().scheduleChore(replicationMetaCleaner); } @@ -1014,8 +1006,9 @@ public class HMaster extends HRegionServer implements MasterServices { LOG.debug("Stopping service threads"); } // Clean up and close up shop - if (this.logCleaner != null) this.logCleaner.cancel(true); - if (this.hfileCleaner != null) this.hfileCleaner.cancel(true); + for (ScheduledChore chore : storageManagerChores) { + if(chore != null) chore.cancel(true); + } if (this.replicationMetaCleaner != null) this.replicationMetaCleaner.cancel(true); if (this.quotaManager != null) this.quotaManager.stop(); if (this.activeMasterManager != null) this.activeMasterManager.stop(); @@ -2453,10 +2446,6 @@ public class HMaster extends HRegionServer implements MasterServices { new HMasterCommandLine(HMaster.class).doMain(args); } - public HFileCleaner getHFileCleaner() { - return this.hfileCleaner; - } - /** * @return the underlying snapshot manager */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java index 1def1f8..d7c28c9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl; -import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; /** @@ -92,7 +91,7 @@ public class MasterStatusServlet extends HttpServlet { boolean showFragmentation = conf.getBoolean( "hbase.master.ui.fragmentation.enabled", false); if (showFragmentation) { - return FSUtils.getTableFragmentation(master); + return master.getMasterStorage().getTableFragmentation(); } else { return null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java deleted file mode 100644 index 891db22..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.hbase.BaseConfigurable; - -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; - -import java.util.Map; - -/** - * Base class for file cleaners which allows subclasses to implement a simple - * isFileDeletable method (which used to be the FileCleanerDelegate contract). - */ -public abstract class BaseFileCleanerDelegate extends BaseConfigurable -implements FileCleanerDelegate { - - @Override - public Iterable getDeletableFiles(Iterable files) { - return Iterables.filter(files, new Predicate() { - @Override - public boolean apply(FileStatus file) { - return isFileDeletable(file); - }}); - } - - @Override - public void init(Map params) { - // subclass could override it if needed. - } - - /** - * Should the master delete the file or keep it? - * @param fStat file status of the file to check - * @return true if the file is deletable, false if not - */ - protected abstract boolean isFileDeletable(FileStatus fStat); - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java deleted file mode 100644 index d2330f9..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * Base class for the hfile cleaning function inside the master. By default, only the - * {@link TimeToLiveHFileCleaner} is called. - *

- * If other effects are needed, implement your own LogCleanerDelegate and add it to the - * configuration "hbase.master.hfilecleaner.plugins", which is a comma-separated list of fully - * qualified class names. The HFileCleaner will build the cleaner chain in - * order the order specified by the configuration. - *

- *

- * For subclasses, setConf will be called exactly once before using the cleaner. - *

- *

- * Since {@link BaseHFileCleanerDelegate HFileCleanerDelegates} are created in - * HFileCleaner by reflection, classes that implements this interface must - * provide a default constructor. - *

- */ -@InterfaceAudience.Private -public abstract class BaseHFileCleanerDelegate extends BaseFileCleanerDelegate { - - private boolean stopped = false; - - @Override - public void stop(String why) { - this.stopped = true; - } - - @Override - public boolean isStopped() { - return this.stopped; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java deleted file mode 100644 index 32e2a7b..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.fs.FileStatus; - -/** - * Base class for the log cleaning function inside the master. By default, two - * cleaners: TimeToLiveLogCleaner and - * ReplicationLogCleaner are called in order. So if other effects - * are needed, implement your own LogCleanerDelegate and add it to the - * configuration "hbase.master.logcleaner.plugins", which is a comma-separated - * list of fully qualified class names. LogsCleaner will add it to the chain. - *

- * HBase ships with LogsCleaner as the default implementation. - *

- * This interface extends Configurable, so setConf needs to be called once - * before using the cleaner. Since LogCleanerDelegates are created in - * LogsCleaner by reflection. Classes that implements this interface should - * provide a default constructor. - */ -@InterfaceAudience.Private -public abstract class BaseLogCleanerDelegate extends BaseFileCleanerDelegate { - - @Override - public boolean isFileDeletable(FileStatus fStat) { - return isLogDeletable(fStat); - } - - /** - * Should the master delete the log or keep it? - *

- * Implementing classes should override {@link #isFileDeletable(FileStatus)} instead. - * @param fStat file status of the file - * @return true if the log is deletable, false (default) if not - */ - @Deprecated - public boolean isLogDeletable(FileStatus fStat) { - return false; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java deleted file mode 100644 index b094507..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java +++ /dev/null @@ -1,294 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.ScheduledChore; -import org.apache.hadoop.hbase.Stoppable; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.ipc.RemoteException; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * Abstract Cleaner that uses a chain of delegates to clean a directory of files - * @param Cleaner delegate class that is dynamically loaded from configuration - */ -public abstract class CleanerChore extends ScheduledChore { - - private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName()); - - private final FileSystem fs; - private final Path oldFileDir; - private final Configuration conf; - protected List cleanersChain; - protected Map params; - - public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf, - FileSystem fs, Path oldFileDir, String confKey) { - this(name, sleepPeriod, s, conf, fs, oldFileDir, confKey, null); - } - - /** - * @param name name of the chore being run - * @param sleepPeriod the period of time to sleep between each run - * @param s the stopper - * @param conf configuration to use - * @param fs handle to the FS - * @param oldFileDir the path to the archived files - * @param confKey configuration key for the classes to instantiate - * @param params members could be used in cleaner - */ - public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf, - FileSystem fs, Path oldFileDir, String confKey, Map params) { - super(name, s, sleepPeriod); - this.fs = fs; - this.oldFileDir = oldFileDir; - this.conf = conf; - this.params = params; - initCleanerChain(confKey); - } - - - /** - * Validate the file to see if it even belongs in the directory. If it is valid, then the file - * will go through the cleaner delegates, but otherwise the file is just deleted. - * @param file full {@link Path} of the file to be checked - * @return true if the file is valid, false otherwise - */ - protected abstract boolean validate(Path file); - - /** - * Instantiate and initialize all the file cleaners set in the configuration - * @param confKey key to get the file cleaner classes from the configuration - */ - private void initCleanerChain(String confKey) { - this.cleanersChain = new LinkedList(); - String[] logCleaners = conf.getStrings(confKey); - if (logCleaners != null) { - for (String className : logCleaners) { - T logCleaner = newFileCleaner(className, conf); - if (logCleaner != null) { - LOG.debug("initialize cleaner=" + className); - this.cleanersChain.add(logCleaner); - } - } - } - } - - /** - * A utility method to create new instances of LogCleanerDelegate based on the class name of the - * LogCleanerDelegate. - * @param className fully qualified class name of the LogCleanerDelegate - * @param conf - * @return the new instance - */ - private T newFileCleaner(String className, Configuration conf) { - try { - Class c = Class.forName(className).asSubclass( - FileCleanerDelegate.class); - @SuppressWarnings("unchecked") - T cleaner = (T) c.newInstance(); - cleaner.setConf(conf); - cleaner.init(this.params); - return cleaner; - } catch (Exception e) { - LOG.warn("Can NOT create CleanerDelegate: " + className, e); - // skipping if can't instantiate - return null; - } - } - - @Override - protected void chore() { - try { - FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir); - checkAndDeleteEntries(files); - } catch (IOException e) { - e = e instanceof RemoteException ? - ((RemoteException)e).unwrapRemoteException() : e; - LOG.warn("Error while cleaning the logs", e); - } - } - - /** - * Loop over the given directory entries, and check whether they can be deleted. - * If an entry is itself a directory it will be recursively checked and deleted itself iff - * all subentries are deleted (and no new subentries are added in the mean time) - * - * @param entries directory entries to check - * @return true if all entries were successfully deleted - */ - private boolean checkAndDeleteEntries(FileStatus[] entries) { - if (entries == null) { - return true; - } - boolean allEntriesDeleted = true; - List files = Lists.newArrayListWithCapacity(entries.length); - for (FileStatus child : entries) { - Path path = child.getPath(); - if (child.isDirectory()) { - // for each subdirectory delete it and all entries if possible - if (!checkAndDeleteDirectory(path)) { - allEntriesDeleted = false; - } - } else { - // collect all files to attempt to delete in one batch - files.add(child); - } - } - if (!checkAndDeleteFiles(files)) { - allEntriesDeleted = false; - } - return allEntriesDeleted; - } - - /** - * Attempt to delete a directory and all files under that directory. Each child file is passed - * through the delegates to see if it can be deleted. If the directory has no children when the - * cleaners have finished it is deleted. - *

- * If new children files are added between checks of the directory, the directory will not - * be deleted. - * @param dir directory to check - * @return true if the directory was deleted, false otherwise. - */ - @VisibleForTesting boolean checkAndDeleteDirectory(Path dir) { - if (LOG.isTraceEnabled()) { - LOG.trace("Checking directory: " + dir); - } - - try { - FileStatus[] children = FSUtils.listStatus(fs, dir); - boolean allChildrenDeleted = checkAndDeleteEntries(children); - - // if the directory still has children, we can't delete it, so we are done - if (!allChildrenDeleted) return false; - } catch (IOException e) { - e = e instanceof RemoteException ? - ((RemoteException)e).unwrapRemoteException() : e; - LOG.warn("Error while listing directory: " + dir, e); - // couldn't list directory, so don't try to delete, and don't return success - return false; - } - - // otherwise, all the children (that we know about) have been deleted, so we should try to - // delete this directory. However, don't do so recursively so we don't delete files that have - // been added since we last checked. - try { - return fs.delete(dir, false); - } catch (IOException e) { - if (LOG.isTraceEnabled()) { - LOG.trace("Couldn't delete directory: " + dir, e); - } - // couldn't delete w/o exception, so we can't return success. - return false; - } - } - - /** - * Run the given files through each of the cleaners to see if it should be deleted, deleting it if - * necessary. - * @param files List of FileStatus for the files to check (and possibly delete) - * @return true iff successfully deleted all files - */ - private boolean checkAndDeleteFiles(List files) { - // first check to see if the path is valid - List validFiles = Lists.newArrayListWithCapacity(files.size()); - List invalidFiles = Lists.newArrayList(); - for (FileStatus file : files) { - if (validate(file.getPath())) { - validFiles.add(file); - } else { - LOG.warn("Found a wrongly formatted file: " + file.getPath() + " - will delete it."); - invalidFiles.add(file); - } - } - - Iterable deletableValidFiles = validFiles; - // check each of the cleaners for the valid files - for (T cleaner : cleanersChain) { - if (cleaner.isStopped() || this.getStopper().isStopped()) { - LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any more files in:" - + this.oldFileDir); - return false; - } - - Iterable filteredFiles = cleaner.getDeletableFiles(deletableValidFiles); - - // trace which cleaner is holding on to each file - if (LOG.isTraceEnabled()) { - ImmutableSet filteredFileSet = ImmutableSet.copyOf(filteredFiles); - for (FileStatus file : deletableValidFiles) { - if (!filteredFileSet.contains(file)) { - LOG.trace(file.getPath() + " is not deletable according to:" + cleaner); - } - } - } - - deletableValidFiles = filteredFiles; - } - - Iterable filesToDelete = Iterables.concat(invalidFiles, deletableValidFiles); - int deletedFileCount = 0; - for (FileStatus file : filesToDelete) { - Path filePath = file.getPath(); - if (LOG.isDebugEnabled()) { - LOG.debug("Removing: " + filePath + " from archive"); - } - try { - boolean success = this.fs.delete(filePath, false); - if (success) { - deletedFileCount++; - } else { - LOG.warn("Attempted to delete:" + filePath - + ", but couldn't. Run cleaner chain and attempt to delete on next pass."); - } - } catch (IOException e) { - e = e instanceof RemoteException ? - ((RemoteException)e).unwrapRemoteException() : e; - LOG.warn("Error while deleting: " + filePath, e); - } - } - - return deletedFileCount == files.size(); - } - - @Override - public void cleanup() { - for (T lc : this.cleanersChain) { - try { - lc.stop("Exiting"); - } catch (Throwable t) { - LOG.warn("Stopping", t); - } - } - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java deleted file mode 100644 index 7a15b96..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.hbase.Stoppable; - -import java.util.Map; - -/** - * General interface for cleaning files from a folder (generally an archive or - * backup folder). These are chained via the {@link CleanerChore} to determine - * if a given file should be deleted. - */ -@InterfaceAudience.Private -public interface FileCleanerDelegate extends Configurable, Stoppable { - - /** - * Determines which of the given files are safe to delete - * @param files files to check for deletion - * @return files that are ok to delete according to this cleaner - */ - Iterable getDeletableFiles(Iterable files); - - - /** - * this method is used to pass some instance into subclass - * */ - void init(Map params); -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java deleted file mode 100644 index 89c316b..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Stoppable; -import org.apache.hadoop.hbase.io.HFileLink; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -/** - * This Chore, every time it runs, will clear the HFiles in the hfile archive - * folder that are deletable for each HFile cleaner in the chain. - */ -@InterfaceAudience.Private -public class HFileCleaner extends CleanerChore { - - public static final String MASTER_HFILE_CLEANER_PLUGINS = "hbase.master.hfilecleaner.plugins"; - - public HFileCleaner(final int period, final Stoppable stopper, Configuration conf, FileSystem fs, - Path directory) { - this(period, stopper, conf, fs, directory, null); - } - - /** - * @param period the period of time to sleep between each run - * @param stopper the stopper - * @param conf configuration to use - * @param fs handle to the FS - * @param directory directory to be cleaned - * @param params params could be used in subclass of BaseHFileCleanerDelegate - */ - public HFileCleaner(final int period, final Stoppable stopper, Configuration conf, FileSystem fs, - Path directory, Map params) { - super("HFileCleaner", period, stopper, conf, fs, - directory, MASTER_HFILE_CLEANER_PLUGINS, params); - } - - @Override - protected boolean validate(Path file) { - if (HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent())) { - return true; - } - return StoreFileInfo.validateStoreFileName(file.getName()); - } - - /** - * Exposed for TESTING! - */ - public List getDelegatesForTesting() { - return this.cleanersChain; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java deleted file mode 100644 index 328a269..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.io.HFileLink; -import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.util.FSUtils; - -/** - * HFileLink cleaner that determines if a hfile should be deleted. - * HFiles can be deleted only if there're no links to them. - * - * When a HFileLink is created a back reference file is created in: - * /hbase/archive/table/region/cf/.links-hfile/ref-region.ref-table - * To check if the hfile can be deleted the back references folder must be empty. - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class HFileLinkCleaner extends BaseHFileCleanerDelegate { - private static final Log LOG = LogFactory.getLog(HFileLinkCleaner.class); - - private FileSystem fs = null; - - @Override - public synchronized boolean isFileDeletable(FileStatus fStat) { - if (this.fs == null) return false; - Path filePath = fStat.getPath(); - // HFile Link is always deletable - if (HFileLink.isHFileLink(filePath)) return true; - - // If the file is inside a link references directory, means that it is a back ref link. - // The back ref can be deleted only if the referenced file doesn't exists. - Path parentDir = filePath.getParent(); - if (HFileLink.isBackReferencesDir(parentDir)) { - Path hfilePath = null; - try { - // Also check if the HFile is in the HBASE_TEMP_DIRECTORY; this is where the referenced - // file gets created when cloning a snapshot. - hfilePath = HFileLink.getHFileFromBackReference( - new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY), filePath); - if (fs.exists(hfilePath)) { - return false; - } - // check whether the HFileLink still exists in mob dir. - hfilePath = HFileLink.getHFileFromBackReference(MobUtils.getMobHome(getConf()), filePath); - if (fs.exists(hfilePath)) { - return false; - } - hfilePath = HFileLink.getHFileFromBackReference(FSUtils.getRootDir(getConf()), filePath); - return !fs.exists(hfilePath); - } catch (IOException e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Couldn't verify if the referenced file still exists, keep it just in case: " - + hfilePath); - } - return false; - } - } - - // HFile is deletable only if has no links - Path backRefDir = null; - try { - backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName()); - return FSUtils.listStatus(fs, backRefDir) == null; - } catch (IOException e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Couldn't get the references, not deleting file, just in case. filePath=" - + filePath + ", backRefDir=" + backRefDir); - } - return false; - } - } - - @Override - public synchronized void setConf(Configuration conf) { - super.setConf(conf); - - // setup filesystem - try { - this.fs = FileSystem.get(this.getConf()); - } catch (IOException e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Couldn't instantiate the file system, not deleting file, just in case. " - + FileSystem.FS_DEFAULT_NAME_KEY + "=" - + getConf().get(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS)); - } - } - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java deleted file mode 100644 index 58da25a..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Stoppable; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; - -/** - * This Chore, every time it runs, will attempt to delete the WALs in the old logs folder. The WAL - * is only deleted if none of the cleaner delegates says otherwise. - * @see BaseLogCleanerDelegate - */ -@InterfaceAudience.Private -public class LogCleaner extends CleanerChore { - private static final Log LOG = LogFactory.getLog(LogCleaner.class.getName()); - - /** - * @param p the period of time to sleep between each run - * @param s the stopper - * @param conf configuration to use - * @param fs handle to the FS - * @param oldLogDir the path to the archived logs - */ - public LogCleaner(final int p, final Stoppable s, Configuration conf, FileSystem fs, - Path oldLogDir) { - super("LogsCleaner", p, s, conf, fs, oldLogDir, HBASE_MASTER_LOGCLEANER_PLUGINS); - } - - @Override - protected boolean validate(Path file) { - return AbstractFSWALProvider.validateWALFilename(file.getName()); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java deleted file mode 100644 index e821b2e..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - -/** - * HFile cleaner that uses the timestamp of the hfile to determine if it should be deleted. By - * default they are allowed to live for {@value #DEFAULT_TTL} - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class TimeToLiveHFileCleaner extends BaseHFileCleanerDelegate { - - private static final Log LOG = LogFactory.getLog(TimeToLiveHFileCleaner.class.getName()); - public static final String TTL_CONF_KEY = "hbase.master.hfilecleaner.ttl"; - // default ttl = 5 minutes - public static final long DEFAULT_TTL = 60000 * 5; - // Configured time a hfile can be kept after it was moved to the archive - private long ttl; - - @Override - public void setConf(Configuration conf) { - this.ttl = conf.getLong(TTL_CONF_KEY, DEFAULT_TTL); - super.setConf(conf); - } - - @Override - public boolean isFileDeletable(FileStatus fStat) { - long currentTime = EnvironmentEdgeManager.currentTime(); - long time = fStat.getModificationTime(); - long life = currentTime - time; - if (LOG.isTraceEnabled()) { - LOG.trace("HFile life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: " - + time); - } - if (life < 0) { - LOG.warn("Found a hfile (" + fStat.getPath() + ") newer than current time (" + currentTime - + " < " + time + "), probably a clock skew"); - return false; - } - return life > ttl; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java deleted file mode 100644 index e46d6e1..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; - -/** - * Log cleaner that uses the timestamp of the wal to determine if it should - * be deleted. By default they are allowed to live for 10 minutes. - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -public class TimeToLiveLogCleaner extends BaseLogCleanerDelegate { - private static final Log LOG = LogFactory.getLog(TimeToLiveLogCleaner.class.getName()); - // Configured time a log can be kept after it was closed - private long ttl; - private boolean stopped = false; - - @Override - public boolean isLogDeletable(FileStatus fStat) { - long currentTime = EnvironmentEdgeManager.currentTime(); - long time = fStat.getModificationTime(); - long life = currentTime - time; - - if (LOG.isTraceEnabled()) { - LOG.trace("Log life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: " - + time); - } - if (life < 0) { - LOG.warn("Found a log (" + fStat.getPath() + ") newer than current time (" + currentTime - + " < " + time + "), probably a clock skew"); - return false; - } - return life > ttl; - } - - @Override - public void setConf(Configuration conf) { - super.setConf(conf); - this.ttl = conf.getLong("hbase.master.logcleaner.ttl", 600000); - } - - - @Override - public void stop(String why) { - this.stopped = true; - } - - @Override - public boolean isStopped() { - return this.stopped; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java deleted file mode 100644 index f80d962..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java +++ /dev/null @@ -1,389 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.snapshot; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.locks.ReentrantLock; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Stoppable; -import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; -import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; -import org.apache.hadoop.hbase.util.FSUtils; - -/** - * Intelligently keep track of all the files for all the snapshots. - *

- * A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache - * miss the directory modification time is used to ensure that we don't rescan directories that we - * already have in cache. We only check the modification times of the snapshot directories - * (/hbase/.snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache. - *

- * New snapshots will be added to the cache and deleted snapshots will be removed when we refresh - * the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself, - * we will ignore updates to that snapshot's files. - *

- * This is sufficient because each snapshot has its own directory and is added via an atomic rename - * once, when the snapshot is created. We don't need to worry about the data in the snapshot - * being run. - *

- * Further, the cache is periodically refreshed ensure that files in snapshots that were deleted are - * also removed from the cache. - *

- * A {@link SnapshotFileCache.SnapshotFileInspector} must be passed when creating this to - * allow extraction of files under /hbase/.snapshot/[snapshot name] directory, for each snapshot. - * This allows you to only cache files under, for instance, all the logs in the .logs directory or - * all the files under all the regions. - *

- * this also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid - * snapshots and will attempt to cache files from those snapshots as well. - *

- * Queries about a given file are thread-safe with respect to multiple queries and cache refreshes. - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class SnapshotFileCache implements Stoppable { - interface SnapshotFileInspector { - /** - * Returns a collection of file names needed by the snapshot. - * @param snapshotDir {@link Path} to the snapshot directory to scan. - * @return the collection of file names needed by the snapshot. - */ - Collection filesUnderSnapshot(final Path snapshotDir) throws IOException; - } - - private static final Log LOG = LogFactory.getLog(SnapshotFileCache.class); - private volatile boolean stop = false; - private final FileSystem fs; - private final SnapshotFileInspector fileInspector; - private final Path snapshotDir; - private final Set cache = new HashSet(); - /** - * This is a helper map of information about the snapshot directories so we don't need to rescan - * them if they haven't changed since the last time we looked. - */ - private final Map snapshots = - new HashMap(); - private final Timer refreshTimer; - - private long lastModifiedTime = Long.MIN_VALUE; - - /** - * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the - * filesystem. - *

- * Immediately loads the file cache. - * @param conf to extract the configured {@link FileSystem} where the snapshots are stored and - * hbase root directory - * @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed - * @param refreshThreadName name of the cache refresh thread - * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files. - * @throws IOException if the {@link FileSystem} or root directory cannot be loaded - */ - public SnapshotFileCache(Configuration conf, long cacheRefreshPeriod, String refreshThreadName, - SnapshotFileInspector inspectSnapshotFiles) throws IOException { - this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), 0, cacheRefreshPeriod, - refreshThreadName, inspectSnapshotFiles); - } - - /** - * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the - * filesystem - * @param fs {@link FileSystem} where the snapshots are stored - * @param rootDir hbase root directory - * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed - * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed - * @param refreshThreadName name of the cache refresh thread - * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files. - */ - public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod, - long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) { - this.fs = fs; - this.fileInspector = inspectSnapshotFiles; - this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); - // periodically refresh the file cache to make sure we aren't superfluously saving files. - this.refreshTimer = new Timer(refreshThreadName, true); - this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay, - cacheRefreshPeriod); - } - - /** - * Trigger a cache refresh, even if its before the next cache refresh. Does not affect pending - * cache refreshes. - *

- * Blocks until the cache is refreshed. - *

- * Exposed for TESTING. - */ - public void triggerCacheRefreshForTesting() { - try { - SnapshotFileCache.this.refreshCache(); - } catch (IOException e) { - LOG.warn("Failed to refresh snapshot hfile cache!", e); - } - LOG.debug("Current cache:" + cache); - } - - /** - * Check to see if any of the passed file names is contained in any of the snapshots. - * First checks an in-memory cache of the files to keep. If its not in the cache, then the cache - * is refreshed and the cache checked again for that file. - * This ensures that we never return files that exist. - *

- * Note this may lead to periodic false positives for the file being referenced. Periodically, the - * cache is refreshed even if there are no requests to ensure that the false negatives get removed - * eventually. For instance, suppose you have a file in the snapshot and it gets loaded into the - * cache. Then at some point later that snapshot is deleted. If the cache has not been refreshed - * at that point, cache will still think the file system contains that file and return - * true, even if it is no longer present (false positive). However, if the file never was - * on the filesystem, we will never find it and always return false. - * @param files file to check, NOTE: Relies that files are loaded from hdfs before method - * is called (NOT LAZY) - * @return unReferencedFiles the collection of files that do not have snapshot references - * @throws IOException if there is an unexpected error reaching the filesystem. - */ - // XXX this is inefficient to synchronize on the method, when what we really need to guard against - // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the - // cache, but that seems overkill at the moment and isn't necessarily a bottleneck. - public synchronized Iterable getUnreferencedFiles(Iterable files, - final SnapshotManager snapshotManager) - throws IOException { - List unReferencedFiles = Lists.newArrayList(); - List snapshotsInProgress = null; - boolean refreshed = false; - for (FileStatus file : files) { - String fileName = file.getPath().getName(); - if (!refreshed && !cache.contains(fileName)) { - refreshCache(); - refreshed = true; - } - if (cache.contains(fileName)) { - continue; - } - if (snapshotsInProgress == null) { - snapshotsInProgress = getSnapshotsInProgress(snapshotManager); - } - if (snapshotsInProgress.contains(fileName)) { - continue; - } - unReferencedFiles.add(file); - } - return unReferencedFiles; - } - - private synchronized void refreshCache() throws IOException { - long lastTimestamp = Long.MAX_VALUE; - boolean hasChanges = false; - - // get the status of the snapshots directory and check if it is has changes - try { - FileStatus dirStatus = fs.getFileStatus(snapshotDir); - lastTimestamp = dirStatus.getModificationTime(); - hasChanges |= (lastTimestamp >= lastModifiedTime); - } catch (FileNotFoundException e) { - if (this.cache.size() > 0) { - LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist"); - } - return; - } - - // get the status of the snapshots temporary directory and check if it has changes - // The top-level directory timestamp is not updated, so we have to check the inner-level. - try { - Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME); - FileStatus tempDirStatus = fs.getFileStatus(snapshotTmpDir); - lastTimestamp = Math.min(lastTimestamp, tempDirStatus.getModificationTime()); - hasChanges |= (lastTimestamp >= lastModifiedTime); - if (!hasChanges) { - FileStatus[] tmpSnapshots = FSUtils.listStatus(fs, snapshotDir); - if (tmpSnapshots != null) { - for (FileStatus dirStatus: tmpSnapshots) { - lastTimestamp = Math.min(lastTimestamp, dirStatus.getModificationTime()); - } - hasChanges |= (lastTimestamp >= lastModifiedTime); - } - } - } catch (FileNotFoundException e) { - // Nothing todo, if the tmp dir is empty - } - - // if the snapshot directory wasn't modified since we last check, we are done - if (!hasChanges) { - return; - } - - // directory was modified, so we need to reload our cache - // there could be a slight race here where we miss the cache, check the directory modification - // time, then someone updates the directory, causing us to not scan the directory again. - // However, snapshot directories are only created once, so this isn't an issue. - - // 1. update the modified time - this.lastModifiedTime = lastTimestamp; - - // 2.clear the cache - this.cache.clear(); - Map known = new HashMap(); - - // 3. check each of the snapshot directories - FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir); - if (snapshots == null) { - // remove all the remembered snapshots because we don't have any left - if (LOG.isDebugEnabled() && this.snapshots.size() > 0) { - LOG.debug("No snapshots on-disk, cache empty"); - } - this.snapshots.clear(); - return; - } - - // 3.1 iterate through the on-disk snapshots - for (FileStatus snapshot : snapshots) { - String name = snapshot.getPath().getName(); - // its not the tmp dir, - if (!name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) { - SnapshotDirectoryInfo files = this.snapshots.remove(name); - // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the - // files the latter could occur where I create a snapshot, then delete it, and then make a - // new snapshot with the same name. We will need to update the cache the information from - // that new snapshot, even though it has the same name as the files referenced have - // probably changed. - if (files == null || files.hasBeenModified(snapshot.getModificationTime())) { - // get all files for the snapshot and create a new info - Collection storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath()); - files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles); - } - // 3.2 add all the files to cache - this.cache.addAll(files.getFiles()); - known.put(name, files); - } - } - - // 4. set the snapshots we are tracking - this.snapshots.clear(); - this.snapshots.putAll(known); - } - - @VisibleForTesting List getSnapshotsInProgress( - final SnapshotManager snapshotManager) throws IOException { - List snapshotInProgress = Lists.newArrayList(); - // only add those files to the cache, but not to the known snapshots - Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME); - // only add those files to the cache, but not to the known snapshots - FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir); - if (running != null) { - for (FileStatus run : running) { - ReentrantLock lock = null; - if (snapshotManager != null) { - lock = snapshotManager.getLocks().acquireLock(run.getPath().getName()); - } - try { - snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath())); - } catch (CorruptedSnapshotException e) { - // See HBASE-16464 - if (e.getCause() instanceof FileNotFoundException) { - // If the snapshot is corrupt, we will delete it - fs.delete(run.getPath(), true); - LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause()); - } else { - throw e; - } - } finally { - if (lock != null) { - lock.unlock(); - } - } - } - } - return snapshotInProgress; - } - - /** - * Simple helper task that just periodically attempts to refresh the cache - */ - public class RefreshCacheTask extends TimerTask { - @Override - public void run() { - try { - SnapshotFileCache.this.refreshCache(); - } catch (IOException e) { - LOG.warn("Failed to refresh snapshot hfile cache!", e); - } - } - } - - @Override - public void stop(String why) { - if (!this.stop) { - this.stop = true; - this.refreshTimer.cancel(); - } - - } - - @Override - public boolean isStopped() { - return this.stop; - } - - /** - * Information about a snapshot directory - */ - private static class SnapshotDirectoryInfo { - long lastModified; - Collection files; - - public SnapshotDirectoryInfo(long mtime, Collection files) { - this.lastModified = mtime; - this.files = files; - } - - /** - * @return the hfiles in the snapshot when this was made. - */ - public Collection getFiles() { - return this.files; - } - - /** - * Check if the snapshot directory has been modified - * @param mtime current modification time of the directory - * @return true if it the modification time of the directory is newer time when we - * created this - */ - public boolean hasBeenModified(long mtime) { - return this.lastModified < mtime; - } - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java deleted file mode 100644 index 2fdbd55..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.snapshot; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate; -import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; -import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; -import org.apache.hadoop.hbase.util.FSUtils; - -/** - * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase - * tables. - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) -@InterfaceStability.Evolving -public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate { - private static final Log LOG = LogFactory.getLog(SnapshotHFileCleaner.class); - - /** - * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in - * snapshots (ms) - */ - public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY = - "hbase.master.hfilecleaner.plugins.snapshot.period"; - - /** Refresh cache, by default, every 5 minutes */ - private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000; - - /** File cache for HFiles in the completed and currently running snapshots */ - private SnapshotFileCache cache; - - private MasterServices master; - - @Override - public synchronized Iterable getDeletableFiles(Iterable files) { - try { - return cache.getUnreferencedFiles(files, master.getSnapshotManager()); - } catch (CorruptedSnapshotException cse) { - LOG.debug("Corrupted in-progress snapshot file exception, ignored ", cse); - } catch (IOException e) { - LOG.error("Exception while checking if files were valid, keeping them just in case.", e); - } - return Collections.emptyList(); - } - - @Override - public void init(Map params) { - if (params.containsKey(HMaster.MASTER)) { - this.master = (MasterServices) params.get(HMaster.MASTER); - } - } - - @Override - protected boolean isFileDeletable(FileStatus fStat) { - return false; - } - - public void setConf(final Configuration conf) { - super.setConf(conf); - try { - long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, - DEFAULT_HFILE_CACHE_REFRESH_PERIOD); - final FileSystem fs = FSUtils.getCurrentFileSystem(conf); - Path rootDir = FSUtils.getRootDir(conf); - cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod, - "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() { - public Collection filesUnderSnapshot(final Path snapshotDir) - throws IOException { - return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir); - } - }); - } catch (IOException e) { - LOG.error("Failed to create cleaner util", e); - } - } - - - @Override - public void stop(String why) { - this.cache.stop(why); - } - - @Override - public boolean isStopped() { - return this.cache.isStopped(); - } - - /** - * Exposed for Testing! - * @return the cache of all hfiles - */ - public SnapshotFileCache getFileCacheForTesting() { - return this.cache; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index a2c0ea0..75a1a17 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -20,13 +20,10 @@ package org.apache.hadoop.hbase.master.snapshot; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ThreadPoolExecutor; import org.apache.commons.logging.Log; @@ -54,8 +51,6 @@ import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.MetricsMaster; import org.apache.hadoop.hbase.master.SnapshotSentinel; -import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; -import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; import org.apache.hadoop.hbase.master.procedure.CloneSnapshotProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.RestoreSnapshotProcedure; @@ -1054,15 +1049,6 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false); boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled); - // Extract cleaners from conf - Set hfileCleaners = new HashSet(); - String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS); - if (cleaners != null) Collections.addAll(hfileCleaners, cleaners); - - Set logCleaners = new HashSet(); - cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS); - if (cleaners != null) Collections.addAll(logCleaners, cleaners); - // check if an older version of snapshot directory was present Path oldSnapshotDir = new Path(((LegacyPathIdentifier) ms.getRootContainer()).path, HConstants .OLD_SNAPSHOT_DIR_NAME); @@ -1077,20 +1063,10 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable // otherwise we still need to check if cleaners are enabled or not and verify // that there're no snapshot in the .snapshot folder. if (snapshotEnabled) { - // Inject snapshot cleaners, if snapshot.enable is true - hfileCleaners.add(SnapshotHFileCleaner.class.getName()); - hfileCleaners.add(HFileLinkCleaner.class.getName()); - - // Set cleaners conf - conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, - hfileCleaners.toArray(new String[hfileCleaners.size()])); - conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, - logCleaners.toArray(new String[logCleaners.size()])); + ms.enableSnapshots(); } else { // Verify if cleaners are present - snapshotEnabled = - hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) && - hfileCleaners.contains(HFileLinkCleaner.class.getName()); + snapshotEnabled = ms.isSnapshotsEnabled(); // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set. if (snapshotEnabled) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ServerProtobufUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ServerProtobufUtil.java index 416ffee..cf2b630 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ServerProtobufUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ServerProtobufUtil.java @@ -17,134 +17,14 @@ */ package org.apache.hadoop.hbase.protobuf; -// TODO remove unused imports -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InterruptedIOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.nio.ByteBuffer; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.util.NavigableSet; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellScanner; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.ClusterId; -import org.apache.hadoop.hbase.ClusterStatus; -import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseIOException; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; + import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.ServerLoad; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.TagUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Append; -import org.apache.hadoop.hbase.client.CompactionState; -import org.apache.hadoop.hbase.client.Consistency; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Durability; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Increment; -import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.RegionLoadStats; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.SnapshotType; -import org.apache.hadoop.hbase.client.metrics.ScanMetrics; -import org.apache.hadoop.hbase.client.security.SecurityCapability; -import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.filter.ByteArrayComparable; -import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.fs.StorageIdentifier; import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier; -import org.apache.hadoop.hbase.io.LimitInputStream; -import org.apache.hadoop.hbase.io.TimeRange; -import org.apache.hadoop.hbase.master.RegionState; -import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos; -import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest; -import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos; -import org.apache.hadoop.hbase.protobuf.generated.CellProtos; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest; -import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos; -import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.LiveServerInfo; -import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionInTransition; -import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad; -import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; -import org.apache.hadoop.hbase.protobuf.generated.FSProtos.HBaseVersionFileContent; -import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilySchema; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema; -import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService; -import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; -import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest; import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor; import org.apache.hadoop.hbase.util.ByteStringer; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StorefileRefresherChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StorefileRefresherChore.java index a2a0dcc..2dcb1ae 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StorefileRefresherChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StorefileRefresherChore.java @@ -28,7 +28,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner; +import org.apache.hadoop.hbase.fs.legacy.cleaner.TimeToLiveHFileCleaner; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.StringUtils; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java index 4c86244..5b94c4b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java @@ -28,10 +28,9 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate; -import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; +import org.apache.hadoop.hbase.fs.legacy.cleaner.BaseHFileCleanerDelegate; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner; import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationQueuesClient; import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java index 074c113..13aba37 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java @@ -26,9 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate; -import org.apache.hadoop.hbase.replication.ReplicationException; +import org.apache.hadoop.hbase.fs.legacy.cleaner.BaseLogCleanerDelegate; import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationQueuesClient; import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments; @@ -39,9 +37,7 @@ import java.util.List; import java.util.Set; import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; import org.apache.zookeeper.KeeperException; /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java index 741065a..a4364eb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java @@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index 0eacc2b..1d5e75f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -1050,63 +1050,6 @@ public abstract class FSUtils { return blocksDistribution; } - // TODO move this method OUT of FSUtils. No dependencies to HMaster - /** - * Returns the total overall fragmentation percentage. Includes hbase:meta and - * -ROOT- as well. - * - * @param master The master defining the HBase root and file system. - * @return A map for each table and its percentage. - * @throws IOException When scanning the directory fails. - */ - public static int getTotalTableFragmentation(final HMaster master) - throws IOException { - Map map = getTableFragmentation(master); - return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1; - } - - /** - * Runs through the HBase rootdir and checks how many stores for each table - * have more than one file in them. Checks -ROOT- and hbase:meta too. The total - * percentage across all tables is stored under the special key "-TOTAL-". - * - * @param master The master defining the HBase root and file system. - * @return A map for each table and its percentage. - * - * @throws IOException When scanning the directory fails. - */ - public static Map getTableFragmentation(final HMaster master) - throws IOException { - final Map frags = new HashMap(); - int cfCountTotal = 0; - int cfFragTotal = 0; - - MasterStorage ms = master.getMasterStorage(); - for (TableName table: ms.getTables()) { - int cfCount = 0; - int cfFrag = 0; - for (HRegionInfo hri: ms.getRegions(table)) { - RegionStorage rfs = ms.getRegionStorage(hri); - final Collection families = rfs.getFamilies(); - for (String family: families) { - cfCount++; - cfCountTotal++; - if (rfs.getStoreFiles(family).size() > 1) { - cfFrag++; - cfFragTotal++; - } - } - } - // compute percentage per table and store in result list - frags.put(table.getNameAsString(), - cfCount == 0? 0: Math.round((float) cfFrag / cfCount * 100)); - } - // set overall percentage for all tables - frags.put("-TOTAL-", - cfCountTotal == 0? 0: Math.round((float) cfFragTotal / cfCountTotal * 100)); - return frags; - } - /** * Returns the {@link org.apache.hadoop.fs.Path} object representing the table directory under * path rootdir diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 86b70c7..93df9dd 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -25,7 +25,6 @@ import="java.util.List" import="java.util.LinkedHashMap" import="java.util.Map" - import="java.util.Set" import="java.util.Collection" import="java.util.Collections" import="java.util.Comparator" @@ -40,7 +39,6 @@ import="org.apache.hadoop.hbase.master.HMaster" import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator" import="org.apache.hadoop.hbase.util.Bytes" - import="org.apache.hadoop.hbase.util.FSUtils" import="org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos" import="org.apache.hadoop.hbase.protobuf.generated.HBaseProtos" import="org.apache.hadoop.hbase.TableName" @@ -69,7 +67,7 @@ HConstants.DEFAULT_META_REPLICA_NUM); Map frags = null; if (showFragmentation) { - frags = FSUtils.getTableFragmentation(master); + frags = master.getMasterStorage().getTableFragmentation(); } String action = request.getParameter("action"); String key = request.getParameter("key"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 467e903..45d8aae 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -81,6 +81,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -1153,6 +1154,19 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { MiniHBaseCluster.class.getName()); } + public HFileCleaner getHFileCleanerChore() { + HFileCleaner hfileCleaner = null; + HMaster master = getMiniHBaseCluster().getMaster(); + Iterable chores = master.getMasterStorage().getChores(master, null); + for (ScheduledChore chore : chores) { + if (chore instanceof HFileCleaner) { + hfileCleaner = (HFileCleaner) chore; + break; + } + } + return hfileCleaner; + } + /** * Stops mini hbase, zk, and hdfs clusters. * @throws IOException diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java index 305d2e8..8321f29 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java @@ -33,13 +33,13 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hbase.backup.HFileArchiver; import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; @@ -48,7 +48,6 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.StoppableImplementation; import org.junit.After; @@ -59,8 +58,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; /** - * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up - * a region + * Test that the {@link HFileArchiver} correctly removes all the + * parts of a region when cleaning up a region */ @Category({MediumTests.class, MiscTests.class}) public class TestHFileArchiving { @@ -78,7 +77,10 @@ public class TestHFileArchiving { UTIL.startMiniCluster(); // We don't want the cleaner to remove files. The tests do that. - UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().cancel(true); + HFileCleaner chore = UTIL.getHFileCleanerChore(); + if (chore != null) { + chore.cancel(true); + } } private static void setupConf(Configuration conf) { @@ -386,8 +388,7 @@ public class TestHFileArchiving { try { // Try to archive the file - HFileArchiver.archiveRegion(fs, rootDir, - sourceRegionDir.getParent(), sourceRegionDir); + HFileArchiver.archiveRegion(fs, rootDir, sourceRegionDir.getParent(), sourceRegionDir); // The archiver succeded, the file is no longer in the original location // but it's in the archive location. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java index 64139ee..0f920d7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java @@ -42,8 +42,8 @@ import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate; -import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; +import org.apache.hadoop.hbase.fs.legacy.cleaner.BaseHFileCleanerDelegate; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner; import org.apache.hadoop.hbase.regionserver.CompactedHFilesDischarger; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.Region; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java index 65a67d0..500b26f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; @@ -249,7 +250,10 @@ public class TestCloneSnapshotFromClient { // ========================================================================== private void waitCleanerRun() throws InterruptedException { - TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting(); + HFileCleaner chore = TEST_UTIL.getHFileCleanerChore(); + if (chore != null) { + chore.choreForTesting(); + } } protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java index 9b4206c..0658904 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.fs.MasterStorage; import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; @@ -298,7 +299,10 @@ public class TestRestoreSnapshotFromClient { // Helpers // ========================================================================== private void waitCleanerRun() throws InterruptedException { - TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting(); + HFileCleaner chore = TEST_UTIL.getHFileCleanerChore(); + if (chore != null) { + chore.choreForTesting(); + } } private Set getFamiliesFromFS(final TableName tableName) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestCleanerChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestCleanerChore.java new file mode 100644 index 0000000..86684d1 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestCleanerChore.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.StoppableImplementation; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +@Category({MasterTests.class, SmallTests.class}) +public class TestCleanerChore { + + private static final Log LOG = LogFactory.getLog(TestCleanerChore.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @After + public void cleanup() throws Exception { + // delete and recreate the test directory, ensuring a clean test dir between tests + UTIL.cleanupTestDir(); +} + + + @Test + public void testSavesFilesOnRequest() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, NeverDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + + // create the directory layout in the directory to clean + Path parent = new Path(testDir, "parent"); + Path file = new Path(parent, "someFile"); + fs.mkdirs(parent); + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + + // run the chore + chore.chore(); + + // verify all the files got deleted + assertTrue("File didn't get deleted", fs.exists(file)); + assertTrue("Empty directory didn't get deleted", fs.exists(parent)); + } + + @Test + public void testDeletesEmptyDirectories() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + + // create the directory layout in the directory to clean + Path parent = new Path(testDir, "parent"); + Path child = new Path(parent, "child"); + Path emptyChild = new Path(parent, "emptyChild"); + Path file = new Path(child, "someFile"); + fs.mkdirs(child); + fs.mkdirs(emptyChild); + // touch a new file + fs.create(file).close(); + // also create a file in the top level directory + Path topFile = new Path(testDir, "topFile"); + fs.create(topFile).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + assertTrue("Test file didn't get created.", fs.exists(topFile)); + + // run the chore + chore.chore(); + + // verify all the files got deleted + assertFalse("File didn't get deleted", fs.exists(topFile)); + assertFalse("File didn't get deleted", fs.exists(file)); + assertFalse("Empty directory didn't get deleted", fs.exists(child)); + assertFalse("Empty directory didn't get deleted", fs.exists(parent)); + } + + /** + * Test to make sure that we don't attempt to ask the delegate whether or not we should preserve a + * directory. + * @throws Exception on failure + */ + @Test + public void testDoesNotCheckDirectories() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + // spy on the delegate to ensure that we don't check for directories + AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); + AlwaysDelete spy = Mockito.spy(delegate); + chore.cleanersChain.set(0, spy); + + // create the directory layout in the directory to clean + Path parent = new Path(testDir, "parent"); + Path file = new Path(parent, "someFile"); + fs.mkdirs(parent); + assertTrue("Test parent didn't get created.", fs.exists(parent)); + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + + FileStatus fStat = fs.getFileStatus(parent); + chore.chore(); + // make sure we never checked the directory + Mockito.verify(spy, Mockito.never()).isFileDeletable(fStat); + Mockito.reset(spy); + } + + @Test + public void testStoppedCleanerDoesNotDeleteFiles() throws Exception { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + Path testDir = UTIL.getDataTestDir(); + FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + + // also create a file in the top level directory + Path topFile = new Path(testDir, "topFile"); + fs.create(topFile).close(); + assertTrue("Test file didn't get created.", fs.exists(topFile)); + + // stop the chore + stop.stop("testing stop"); + + // run the chore + chore.chore(); + + // test that the file still exists + assertTrue("File got deleted while chore was stopped", fs.exists(topFile)); + } + + /** + * While cleaning a directory, all the files in the directory may be deleted, but there may be + * another file added, in which case the directory shouldn't be deleted. + * @throws IOException on failure + */ + @Test + public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException { + Stoppable stop = new StoppableImplementation(); + Configuration conf = UTIL.getConfiguration(); + final Path testDir = UTIL.getDataTestDir(); + final FileSystem fs = UTIL.getTestFileSystem(); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + // spy on the delegate to ensure that we don't check for directories + AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); + AlwaysDelete spy = Mockito.spy(delegate); + chore.cleanersChain.set(0, spy); + + // create the directory layout in the directory to clean + final Path parent = new Path(testDir, "parent"); + Path file = new Path(parent, "someFile"); + fs.mkdirs(parent); + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + final Path addedFile = new Path(parent, "addedFile"); + + // when we attempt to delete the original file, add another file in the same directory + Mockito.doAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + fs.create(addedFile).close(); + FSUtils.logFileSystemState(fs, testDir, LOG); + return (Boolean) invocation.callRealMethod(); + } + }).when(spy).isFileDeletable(Mockito.any(FileStatus.class)); + + // run the chore + chore.chore(); + + // make sure all the directories + added file exist, but the original file is deleted + assertTrue("Added file unexpectedly deleted", fs.exists(addedFile)); + assertTrue("Parent directory deleted unexpectedly", fs.exists(parent)); + assertFalse("Original file unexpectedly retained", fs.exists(file)); + Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class)); + Mockito.reset(spy); + } + + /** + * The cleaner runs in a loop, where it first checks to see all the files under a directory can be + * deleted. If they all can, then we try to delete the directory. However, a file may be added + * that directory to after the original check. This ensures that we don't accidentally delete that + * directory on and don't get spurious IOExceptions. + *

+ * This was from HBASE-7465. + * @throws Exception on failure + */ + @Test + public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception { + Stoppable stop = new StoppableImplementation(); + // need to use a localutil to not break the rest of the test that runs on the local FS, which + // gets hosed when we start to use a minicluster. + HBaseTestingUtility localUtil = new HBaseTestingUtility(); + Configuration conf = localUtil.getConfiguration(); + final Path testDir = UTIL.getDataTestDir(); + final FileSystem fs = UTIL.getTestFileSystem(); + LOG.debug("Writing test data to: " + testDir); + String confKey = "hbase.test.cleaner.delegates"; + conf.set(confKey, AlwaysDelete.class.getName()); + + AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); + // spy on the delegate to ensure that we don't check for directories + AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); + AlwaysDelete spy = Mockito.spy(delegate); + chore.cleanersChain.set(0, spy); + + // create the directory layout in the directory to clean + final Path parent = new Path(testDir, "parent"); + Path file = new Path(parent, "someFile"); + fs.mkdirs(parent); + // touch a new file + fs.create(file).close(); + assertTrue("Test file didn't get created.", fs.exists(file)); + final Path racyFile = new Path(parent, "addedFile"); + + // when we attempt to delete the original file, add another file in the same directory + Mockito.doAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + fs.create(racyFile).close(); + FSUtils.logFileSystemState(fs, testDir, LOG); + return (Boolean) invocation.callRealMethod(); + } + }).when(spy).isFileDeletable(Mockito.any(FileStatus.class)); + + // attempt to delete the directory, which + if (chore.checkAndDeleteDirectory(parent)) { + throw new Exception( + "Reported success deleting directory, should have failed when adding file mid-iteration"); + } + + // make sure all the directories + added file exist, but the original file is deleted + assertTrue("Added file unexpectedly deleted", fs.exists(racyFile)); + assertTrue("Parent directory deleted unexpectedly", fs.exists(parent)); + assertFalse("Original file unexpectedly retained", fs.exists(file)); + Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class)); + } + + private static class AllValidPaths extends CleanerChore { + + public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs, + Path oldFileDir, String confkey) { + super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey); + } + + // all paths are valid + @Override + protected boolean validate(Path file) { + return true; + } + }; + + public static class AlwaysDelete extends BaseHFileCleanerDelegate { + @Override + public boolean isFileDeletable(FileStatus fStat) { + return true; + } + } + + public static class NeverDelete extends BaseHFileCleanerDelegate { + @Override + public boolean isFileDeletable(FileStatus fStat) { + return false; + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileCleaner.java new file mode 100644 index 0000000..01bc4bf --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileCleaner.java @@ -0,0 +1,263 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({MasterTests.class, MediumTests.class}) +public class TestHFileCleaner { + private static final Log LOG = LogFactory.getLog(TestHFileCleaner.class); + + private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupCluster() throws Exception { + // have to use a minidfs cluster because the localfs doesn't modify file times correctly + UTIL.startMiniDFSCluster(1); + } + + @AfterClass + public static void shutdownCluster() throws IOException { + UTIL.shutdownMiniDFSCluster(); + } + + @Test + public void testTTLCleaner() throws IOException, InterruptedException { + FileSystem fs = UTIL.getDFSCluster().getFileSystem(); + Path root = UTIL.getDataTestDirOnTestFS(); + Path file = new Path(root, "file"); + fs.createNewFile(file); + long createTime = System.currentTimeMillis(); + assertTrue("Test file not created!", fs.exists(file)); + TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner(); + // update the time info for the file, so the cleaner removes it + fs.setTimes(file, createTime - 100, -1); + Configuration conf = UTIL.getConfiguration(); + conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100); + cleaner.setConf(conf); + assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs) + + " with create time:" + createTime, cleaner.isFileDeletable(fs.getFileStatus(file))); + } + + /** + * @param file to check + * @return loggable information about the file + */ + private String getFileStats(Path file, FileSystem fs) throws IOException { + FileStatus status = fs.getFileStatus(file); + return "File" + file + ", mtime:" + status.getModificationTime() + ", atime:" + + status.getAccessTime(); + } + + @Test(timeout = 60 *1000) + public void testHFileCleaning() throws Exception { + final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate(); + String prefix = "someHFileThatWouldBeAUUID"; + Configuration conf = UTIL.getConfiguration(); + // set TTL + long ttl = 2000; + conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, + "org.apache.hadoop.hbase.fs.legacy.cleaner.TimeToLiveHFileCleaner"); + conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); + Server server = new DummyServer(); + Path archivedHfileDir = new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY); + FileSystem fs = FileSystem.get(conf); + HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir); + + // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files + final long createTime = System.currentTimeMillis(); + fs.delete(archivedHfileDir, true); + fs.mkdirs(archivedHfileDir); + // Case 1: 1 invalid file, which should be deleted directly + fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd")); + // Case 2: 1 "recent" file, not even deletable for the first log cleaner + // (TimeToLiveLogCleaner), so we are not going down the chain + LOG.debug("Now is: " + createTime); + for (int i = 1; i < 32; i++) { + // Case 3: old files which would be deletable for the first log cleaner + // (TimeToLiveHFileCleaner), + Path fileName = new Path(archivedHfileDir, (prefix + "." + (createTime + i))); + fs.createNewFile(fileName); + // set the creation time past ttl to ensure that it gets removed + fs.setTimes(fileName, createTime - ttl - 1, -1); + LOG.debug("Creating " + getFileStats(fileName, fs)); + } + + // Case 2: 1 newer file, not even deletable for the first log cleaner + // (TimeToLiveLogCleaner), so we are not going down the chain + Path saved = new Path(archivedHfileDir, prefix + ".00000000000"); + fs.createNewFile(saved); + // set creation time within the ttl + fs.setTimes(saved, createTime - ttl / 2, -1); + LOG.debug("Creating " + getFileStats(saved, fs)); + for (FileStatus stat : fs.listStatus(archivedHfileDir)) { + LOG.debug(stat.getPath().toString()); + } + + assertEquals(33, fs.listStatus(archivedHfileDir).length); + + // set a custom edge manager to handle time checking + EnvironmentEdge setTime = new EnvironmentEdge() { + @Override + public long currentTime() { + return createTime; + } + }; + EnvironmentEdgeManager.injectEdge(setTime); + + // run the chore + cleaner.chore(); + + // ensure we only end up with the saved file + assertEquals(1, fs.listStatus(archivedHfileDir).length); + + for (FileStatus file : fs.listStatus(archivedHfileDir)) { + LOG.debug("Kept hfiles: " + file.getPath().getName()); + } + + // reset the edge back to the original edge + EnvironmentEdgeManager.injectEdge(originalEdge); + } + + @Test + public void testRemovesEmptyDirectories() throws Exception { + Configuration conf = UTIL.getConfiguration(); + // no cleaner policies = delete all files + conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); + Server server = new DummyServer(); + Path archivedHfileDir = new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY); + + // setup the cleaner + FileSystem fs = UTIL.getDFSCluster().getFileSystem(); + HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir); + + // make all the directories for archiving files + Path table = new Path(archivedHfileDir, "table"); + Path region = new Path(table, "regionsomthing"); + Path family = new Path(region, "fam"); + Path file = new Path(family, "file12345"); + fs.mkdirs(family); + if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family); + fs.create(file).close(); + if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file); + + // run the chore to cleanup the files (and the directories above it) + cleaner.chore(); + + // make sure all the parent directories get removed + assertFalse("family directory not removed for empty directory", fs.exists(family)); + assertFalse("region directory not removed for empty directory", fs.exists(region)); + assertFalse("table directory not removed for empty directory", fs.exists(table)); + assertTrue("archive directory", fs.exists(archivedHfileDir)); + } + + static class DummyServer implements Server { + + @Override + public Configuration getConfiguration() { + return UTIL.getConfiguration(); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + try { + return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public CoordinatedStateManager getCoordinatedStateManager() { + return null; + } + + @Override + public ClusterConnection getConnection() { + return null; + } + + @Override + public MetaTableLocator getMetaTableLocator() { + return null; + } + + @Override + public ServerName getServerName() { + return ServerName.valueOf("regionserver,60020,000000"); + } + + @Override + public void abort(String why, Throwable e) { + } + + @Override + public boolean isAborted() { + return false; + } + + @Override + public void stop(String why) { + } + + @Override + public boolean isStopped() { + return false; + } + + @Override + public ChoreService getChoreService() { + return null; + } + + @Override + public ClusterConnection getClusterConnection() { + // TODO Auto-generated method stub + return null; + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileLinkCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileLinkCleaner.java new file mode 100644 index 0000000..998481a --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileLinkCleaner.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestRule; + +/** + * Test the HFileLink Cleaner. + * HFiles with links cannot be deleted until a link is present. + */ +@Category({MasterTests.class, MediumTests.class}) +public class TestHFileLinkCleaner { + @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @Test + public void testHFileLinkCleaning() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); + conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName()); + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = FileSystem.get(conf); + + final TableName tableName = TableName.valueOf("test-table"); + final TableName tableLinkName = TableName.valueOf("test-link"); + final String hfileName = "1234567890"; + final String familyName = "cf"; + + HRegionInfo hri = new HRegionInfo(tableName); + HRegionInfo hriLink = new HRegionInfo(tableLinkName); + + Path archiveDir = HFileArchiveUtil.getArchivePath(conf); + Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, + tableName, hri.getEncodedName(), familyName); + Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, + tableLinkName, hriLink.getEncodedName(), familyName); + + // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf); + Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName); + fs.mkdirs(familyPath); + Path hfilePath = new Path(familyPath, hfileName); + fs.createNewFile(hfilePath); + + // Create link to hfile + Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName, + hriLink.getEncodedName(), familyName); + fs.mkdirs(familyLinkPath); + HFileLink.create(conf, fs, familyLinkPath, hri, hfileName); + Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); + assertTrue(fs.exists(linkBackRefDir)); + FileStatus[] backRefs = fs.listStatus(linkBackRefDir); + assertEquals(1, backRefs.length); + Path linkBackRef = backRefs[0].getPath(); + + // Initialize cleaner + final long ttl = 1000; + conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); + Server server = new DummyServer(); + HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir); + + // Link backref cannot be removed + cleaner.chore(); + assertTrue(fs.exists(linkBackRef)); + assertTrue(fs.exists(hfilePath)); + + // Link backref can be removed + fs.rename(FSUtils.getTableDir(rootDir, tableLinkName), + FSUtils.getTableDir(archiveDir, tableLinkName)); + cleaner.chore(); + assertFalse("Link should be deleted", fs.exists(linkBackRef)); + + // HFile can be removed + Thread.sleep(ttl * 2); + cleaner.chore(); + assertFalse("HFile should be deleted", fs.exists(hfilePath)); + + // Remove everything + for (int i = 0; i < 4; ++i) { + Thread.sleep(ttl * 2); + cleaner.chore(); + } + assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName))); + assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName))); + } + + private static Path getFamilyDirPath (final Path rootDir, final TableName table, + final String region, final String family) { + return new Path(new Path(FSUtils.getTableDir(rootDir, table), region), family); + } + + static class DummyServer implements Server { + + @Override + public Configuration getConfiguration() { + return TEST_UTIL.getConfiguration(); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + try { + return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public CoordinatedStateManager getCoordinatedStateManager() { + return null; + } + + @Override + public ClusterConnection getConnection() { + return null; + } + + @Override + public MetaTableLocator getMetaTableLocator() { + return null; + } + + @Override + public ServerName getServerName() { + return ServerName.valueOf("regionserver,60020,000000"); + } + + @Override + public void abort(String why, Throwable e) {} + + @Override + public boolean isAborted() { + return false; + } + + @Override + public void stop(String why) {} + + @Override + public boolean isStopped() { + return false; + } + + @Override + public ChoreService getChoreService() { + return null; + } + + @Override + public ClusterConnection getClusterConnection() { + // TODO Auto-generated method stub + return null; + } + } +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestLogsCleaner.java new file mode 100644 index 0000000..d313748 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestLogsCleaner.java @@ -0,0 +1,308 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URLEncoder; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.replication.ReplicationFactory; +import org.apache.hadoop.hbase.replication.ReplicationQueues; +import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments; +import org.apache.hadoop.hbase.replication.ReplicationQueuesClientZKImpl; +import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner; +import org.apache.hadoop.hbase.replication.regionserver.Replication; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category({MasterTests.class, MediumTests.class}) +public class TestLogsCleaner { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + @Test + public void testLogCleaning() throws Exception{ + Configuration conf = TEST_UTIL.getConfiguration(); + // set TTL + long ttl = 10000; + conf.setLong("hbase.master.logcleaner.ttl", ttl); + Replication.decorateMasterConfiguration(conf); + Server server = new DummyServer(); + ReplicationQueues repQueues = + ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, server, server.getZooKeeper())); + repQueues.init(server.getServerName().toString()); + final Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(), + HConstants.HREGION_OLDLOGDIR_NAME); + String fakeMachineName = + URLEncoder.encode(server.getServerName().toString(), "UTF8"); + + final FileSystem fs = FileSystem.get(conf); + + // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files + long now = System.currentTimeMillis(); + fs.delete(oldLogDir, true); + fs.mkdirs(oldLogDir); + // Case 1: 2 invalid files, which would be deleted directly + fs.createNewFile(new Path(oldLogDir, "a")); + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a")); + // Case 2: 1 "recent" file, not even deletable for the first log cleaner + // (TimeToLiveLogCleaner), so we are not going down the chain + System.out.println("Now is: " + now); + for (int i = 1; i < 31; i++) { + // Case 3: old files which would be deletable for the first log cleaner + // (TimeToLiveLogCleaner), and also for the second (ReplicationLogCleaner) + Path fileName = new Path(oldLogDir, fakeMachineName + "." + (now - i) ); + fs.createNewFile(fileName); + // Case 4: put 3 old log files in ZK indicating that they are scheduled + // for replication so these files would pass the first log cleaner + // (TimeToLiveLogCleaner) but would be rejected by the second + // (ReplicationLogCleaner) + if (i % (30/3) == 1) { + repQueues.addLog(fakeMachineName, fileName.getName()); + System.out.println("Replication log file: " + fileName); + } + } + + // sleep for sometime to get newer modifcation time + Thread.sleep(ttl); + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now)); + + // Case 2: 1 newer file, not even deletable for the first log cleaner + // (TimeToLiveLogCleaner), so we are not going down the chain + fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000) )); + + for (FileStatus stat : fs.listStatus(oldLogDir)) { + System.out.println(stat.getPath().toString()); + } + + assertEquals(34, fs.listStatus(oldLogDir).length); + + LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir); + + cleaner.chore(); + + // We end up with the current log file, a newer one and the 3 old log + // files which are scheduled for replication + TEST_UTIL.waitFor(1000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return 5 == fs.listStatus(oldLogDir).length; + } + }); + + for (FileStatus file : fs.listStatus(oldLogDir)) { + System.out.println("Kept log files: " + file.getPath().getName()); + } + } + + @Test(timeout=5000) + public void testZnodeCversionChange() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + ReplicationLogCleaner cleaner = new ReplicationLogCleaner(); + cleaner.setConf(conf); + + ReplicationQueuesClientZKImpl rqcMock = Mockito.mock(ReplicationQueuesClientZKImpl.class); + Mockito.when(rqcMock.getQueuesZNodeCversion()).thenReturn(1, 2, 3, 4); + + Field rqc = ReplicationLogCleaner.class.getDeclaredField("replicationQueues"); + rqc.setAccessible(true); + + rqc.set(cleaner, rqcMock); + + // This should return eventually when cversion stabilizes + cleaner.getDeletableFiles(new LinkedList()); + } + + /** + * ReplicationLogCleaner should be able to ride over ZooKeeper errors without + * aborting. + */ + @Test + public void testZooKeeperAbort() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + ReplicationLogCleaner cleaner = new ReplicationLogCleaner(); + + List dummyFiles = Lists.newArrayList( + new FileStatus(100, false, 3, 100, System.currentTimeMillis(), new Path("log1")), + new FileStatus(100, false, 3, 100, System.currentTimeMillis(), new Path("log2")) + ); + + FaultyZooKeeperWatcher faultyZK = + new FaultyZooKeeperWatcher(conf, "testZooKeeperAbort-faulty", null); + try { + faultyZK.init(); + cleaner.setConf(conf, faultyZK); + // should keep all files due to a ConnectionLossException getting the queues znodes + Iterable toDelete = cleaner.getDeletableFiles(dummyFiles); + assertFalse(toDelete.iterator().hasNext()); + assertFalse(cleaner.isStopped()); + } finally { + faultyZK.close(); + } + + // when zk is working both files should be returned + cleaner = new ReplicationLogCleaner(); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testZooKeeperAbort-normal", null); + try { + cleaner.setConf(conf, zkw); + Iterable filesToDelete = cleaner.getDeletableFiles(dummyFiles); + Iterator iter = filesToDelete.iterator(); + assertTrue(iter.hasNext()); + assertEquals(new Path("log1"), iter.next().getPath()); + assertTrue(iter.hasNext()); + assertEquals(new Path("log2"), iter.next().getPath()); + assertFalse(iter.hasNext()); + } finally { + zkw.close(); + } + } + + static class DummyServer implements Server { + + @Override + public Configuration getConfiguration() { + return TEST_UTIL.getConfiguration(); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + try { + return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public CoordinatedStateManager getCoordinatedStateManager() { + return null; + } + + @Override + public ClusterConnection getConnection() { + return null; + } + + @Override + public MetaTableLocator getMetaTableLocator() { + return null; + } + + @Override + public ServerName getServerName() { + return ServerName.valueOf("regionserver,60020,000000"); + } + + @Override + public void abort(String why, Throwable e) {} + + @Override + public boolean isAborted() { + return false; + } + + @Override + public void stop(String why) {} + + @Override + public boolean isStopped() { + return false; + } + + @Override + public ChoreService getChoreService() { + return null; + } + + @Override + public ClusterConnection getClusterConnection() { + // TODO Auto-generated method stub + return null; + } + } + + static class FaultyZooKeeperWatcher extends ZooKeeperWatcher { + private RecoverableZooKeeper zk; + + public FaultyZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable) + throws ZooKeeperConnectionException, IOException { + super(conf, identifier, abortable); + } + + public void init() throws Exception { + this.zk = spy(super.getRecoverableZooKeeper()); + doThrow(new KeeperException.ConnectionLossException()) + .when(zk).getData("/hbase/replication/rs", null, new Stat()); + } + + public RecoverableZooKeeper getRecoverableZooKeeper() { + return zk; + } + } +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestSnapshotFromMaster.java new file mode 100644 index 0000000..8b8592b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestSnapshotFromMaster.java @@ -0,0 +1,394 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.cleaner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotHFileCleaner; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.regionserver.CompactedHFilesDischarger; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test the master-related aspects of a snapshot + */ +@Category({MasterTests.class, MediumTests.class}) +public class TestSnapshotFromMaster { + + private static final Log LOG = LogFactory.getLog(TestSnapshotFromMaster.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final int NUM_RS = 2; + private static Path rootDir; + private static FileSystem fs; + private static HMaster master; + + // for hfile archiving test. + private static Path archiveDir; + private static final byte[] TEST_FAM = Bytes.toBytes("fam"); + private static final TableName TABLE_NAME = + TableName.valueOf("test"); + // refresh the cache every 1/2 second + private static final long cacheRefreshPeriod = 500; + private static final int blockingStoreFiles = 12; + + /** + * Setup the config for the cluster + */ + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(NUM_RS); + fs = UTIL.getDFSCluster().getFileSystem(); + master = UTIL.getMiniHBaseCluster().getMaster(); +// rootDir = master.getMasterStorage().getRootDir(); + archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); + } + + private static void setupConf(Configuration conf) { + // disable the ui + conf.setInt("hbase.regionsever.info.port", -1); + // change the flush size to a small amount, regulating number of store files + conf.setInt("hbase.hregion.memstore.flush.size", 25000); + // so make sure we get a compaction when doing a load, but keep around some + // files in the store + conf.setInt("hbase.hstore.compaction.min", 2); + conf.setInt("hbase.hstore.compactionThreshold", 5); + // block writes if we get to 12 store files + conf.setInt("hbase.hstore.blockingStoreFiles", blockingStoreFiles); + // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner) + conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); + conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, ""); + // Enable snapshot + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod); + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + ConstantSizeRegionSplitPolicy.class.getName()); + conf.setInt("hbase.hfile.compactions.cleaner.interval", 20 * 1000); + + } + + @Before + public void setup() throws Exception { + UTIL.createTable(TABLE_NAME, TEST_FAM); + master.getSnapshotManager().setSnapshotHandlerForTesting(TABLE_NAME, null); + } + + @After + public void tearDown() throws Exception { + UTIL.deleteTable(TABLE_NAME); + SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin()); + SnapshotTestingUtils.deleteArchiveDirectory(UTIL); + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + // NOOP; + } + } + +// /** +// * Test that the contract from the master for checking on a snapshot are valid. +// *

+// *

    +// *
  1. If a snapshot fails with an error, we expect to get the source error.
  2. +// *
  3. If there is no snapshot name supplied, we should get an error.
  4. +// *
  5. If asking about a snapshot has hasn't occurred, you should get an error.
  6. +// *
+// */ +// @Test(timeout = 300000) +// public void testIsDoneContract() throws Exception { +// +// IsSnapshotDoneRequest.Builder builder = IsSnapshotDoneRequest.newBuilder(); +// +// String snapshotName = "asyncExpectedFailureTest"; +// +// // check that we get an exception when looking up snapshot where one hasn't happened +// SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), +// UnknownSnapshotException.class); +// +// // and that we get the same issue, even if we specify a name +// SnapshotDescription desc = SnapshotDescription.newBuilder() +// .setName(snapshotName).setTable(TABLE_NAME.getNameAsString()).build(); +// builder.setSnapshot(desc); +// SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), +// UnknownSnapshotException.class); +// +// // set a mock handler to simulate a snapshot +// DisabledTableSnapshotHandler mockHandler = Mockito.mock(DisabledTableSnapshotHandler.class); +// Mockito.when(mockHandler.getException()).thenReturn(null); +// Mockito.when(mockHandler.getSnapshot()).thenReturn(desc); +// Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true)); +// Mockito.when(mockHandler.getCompletionTimestamp()) +// .thenReturn(EnvironmentEdgeManager.currentTime()); +// +// master.getSnapshotManager() +// .setSnapshotHandlerForTesting(TABLE_NAME, mockHandler); +// +// // if we do a lookup without a snapshot name, we should fail - you should always know your name +// builder = IsSnapshotDoneRequest.newBuilder(); +// SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), +// UnknownSnapshotException.class); +// +// // then do the lookup for the snapshot that it is done +// builder.setSnapshot(desc); +// IsSnapshotDoneResponse response = +// master.getMasterRpcServices().isSnapshotDone(null, builder.build()); +// assertTrue("Snapshot didn't complete when it should have.", response.getDone()); +// +// // now try the case where we are looking for a snapshot we didn't take +// builder.setSnapshot(SnapshotDescription.newBuilder().setName("Not A Snapshot").build()); +// SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), +// UnknownSnapshotException.class); +// +// // then create a snapshot to the fs and make sure that we can find it when checking done +// snapshotName = "completed"; +// Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); +// desc = desc.toBuilder().setName(snapshotName).build(); +// SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshotDir, fs); +// +// builder.setSnapshot(desc); +// response = master.getMasterRpcServices().isSnapshotDone(null, builder.build()); +// assertTrue("Completed, on-disk snapshot not found", response.getDone()); +// } +// +// @Test(timeout = 300000) +// public void testGetCompletedSnapshots() throws Exception { +// // first check when there are no snapshots +// GetCompletedSnapshotsRequest request = GetCompletedSnapshotsRequest.newBuilder().build(); +// GetCompletedSnapshotsResponse response = +// master.getMasterRpcServices().getCompletedSnapshots(null, request); +// assertEquals("Found unexpected number of snapshots", 0, response.getSnapshotsCount()); +// +// // write one snapshot to the fs +// String snapshotName = "completed"; +// Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); +// SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); +// SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); +// +// // check that we get one snapshot +// response = master.getMasterRpcServices().getCompletedSnapshots(null, request); +// assertEquals("Found unexpected number of snapshots", 1, response.getSnapshotsCount()); +// List snapshots = response.getSnapshotsList(); +// List expected = Lists.newArrayList(snapshot); +// assertEquals("Returned snapshots don't match created snapshots", expected, snapshots); +// +// // write a second snapshot +// snapshotName = "completed_two"; +// snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); +// snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); +// SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); +// expected.add(snapshot); +// +// // check that we get one snapshot +// response = master.getMasterRpcServices().getCompletedSnapshots(null, request); +// assertEquals("Found unexpected number of snapshots", 2, response.getSnapshotsCount()); +// snapshots = response.getSnapshotsList(); +// assertEquals("Returned snapshots don't match created snapshots", expected, snapshots); +// } +// +// @Test(timeout = 300000) +// public void testDeleteSnapshot() throws Exception { +// +// String snapshotName = "completed"; +// SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); +// +// DeleteSnapshotRequest request = DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot) +// .build(); +// try { +// master.getMasterRpcServices().deleteSnapshot(null, request); +// fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist"); +// } catch (ServiceException e) { +// LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage()); +// } +// +// // write one snapshot to the fs +// Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); +// SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); +// +// // then delete the existing snapshot,which shouldn't cause an exception to be thrown +// master.getMasterRpcServices().deleteSnapshot(null, request); +// } + + /** + * Test that the snapshot hfile archive cleaner works correctly. HFiles that are in snapshots + * should be retained, while those that are not in a snapshot should be deleted. + * @throws Exception on failure + */ + @Test(timeout = 300000) + public void testSnapshotHFileArchiving() throws Exception { + Admin admin = UTIL.getHBaseAdmin(); + // make sure we don't fail on listing snapshots + SnapshotTestingUtils.assertNoSnapshots(admin); + + // recreate test table with disabled compactions; otherwise compaction may happen before + // snapshot, the call after snapshot will be a no-op and checks will fail + UTIL.deleteTable(TABLE_NAME); + HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); + htd.setCompactionEnabled(false); + UTIL.createTable(htd, new byte[][] { TEST_FAM }, null); + + // load the table + for (int i = 0; i < blockingStoreFiles / 2; i ++) { + UTIL.loadTable(UTIL.getConnection().getTable(TABLE_NAME), TEST_FAM); + UTIL.flush(TABLE_NAME); + } + + // disable the table so we can take a snapshot + admin.disableTable(TABLE_NAME); + htd.setCompactionEnabled(true); + + // take a snapshot of the table + String snapshotName = "snapshot"; + byte[] snapshotNameBytes = Bytes.toBytes(snapshotName); + admin.snapshot(snapshotNameBytes, TABLE_NAME); + + LOG.info("After snapshot File-System state"); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // ensure we only have one snapshot + SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshotNameBytes, TABLE_NAME); + + // enable compactions now + admin.modifyTable(TABLE_NAME, htd); + + // renable the table so we can compact the regions + admin.enableTable(TABLE_NAME); + + // compact the files so we get some archived files for the table we just snapshotted + List regions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + for (HRegion region : regions) { + region.waitForFlushesAndCompactions(); // enable can trigger a compaction, wait for it. + region.compactStores(); // min is 2 so will compact and archive + } + List regionServerThreads = UTIL.getMiniHBaseCluster() + .getRegionServerThreads(); + HRegionServer hrs = null; + for (RegionServerThread rs : regionServerThreads) { + if (!rs.getRegionServer().getOnlineRegions(TABLE_NAME).isEmpty()) { + hrs = rs.getRegionServer(); + break; + } + } + CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, hrs, false); + cleaner.chore(); + LOG.info("After compaction File-System state"); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // make sure the cleaner has run + LOG.debug("Running hfile cleaners"); + ensureHFileCleanersRun(); + LOG.info("After cleaners File-System state: " + rootDir); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // get the snapshot files for the table + Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + Set snapshotHFiles = SnapshotReferenceUtil.getHFileNames( + UTIL.getConfiguration(), fs, snapshotTable); + // check that the files in the archive contain the ones that we need for the snapshot + LOG.debug("Have snapshot hfiles:"); + for (String fileName : snapshotHFiles) { + LOG.debug(fileName); + } + // get the archived files for the table + Collection archives = getHFiles(archiveDir, fs, TABLE_NAME); + + // get the hfiles for the table + Collection hfiles = getHFiles(rootDir, fs, TABLE_NAME); + + // and make sure that there is a proper subset + for (String fileName : snapshotHFiles) { + boolean exist = archives.contains(fileName) || hfiles.contains(fileName); + assertTrue("Archived hfiles " + archives + + " and table hfiles " + hfiles + " is missing snapshot file:" + fileName, exist); + } + + // delete the existing snapshot + admin.deleteSnapshot(snapshotNameBytes); + SnapshotTestingUtils.assertNoSnapshots(admin); + + // make sure that we don't keep around the hfiles that aren't in a snapshot + // make sure we wait long enough to refresh the snapshot hfile + List delegates = UTIL.getHFileCleanerChore().cleanersChain; + for (BaseHFileCleanerDelegate delegate: delegates) { + if (delegate instanceof SnapshotHFileCleaner) { + ((SnapshotHFileCleaner)delegate).getFileCacheForTesting().triggerCacheRefreshForTesting(); + } + } + // run the cleaner again + LOG.debug("Running hfile cleaners"); + ensureHFileCleanersRun(); + LOG.info("After delete snapshot cleaners run File-System state"); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + archives = getHFiles(archiveDir, fs, TABLE_NAME); + assertEquals("Still have some hfiles in the archive, when their snapshot has been deleted.", 0, + archives.size()); + } + + /** + * @return all the HFiles for a given table in the specified dir + * @throws IOException on expected failure + */ + private final Collection getHFiles(Path dir, FileSystem fs, TableName tableName) throws IOException { + Path tableDir = FSUtils.getTableDir(dir, tableName); + return SnapshotTestingUtils.listHFileNames(fs, tableDir); + } + + /** + * Make sure the {@link HFileCleaner HFileCleaners} run at least once + */ + private static void ensureHFileCleanersRun() { + UTIL.getHFileCleanerChore().chore(); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotFileCache.java new file mode 100644 index 0000000..a92a5bc --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotFileCache.java @@ -0,0 +1,288 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test that we correctly reload the cache, filter directories, etc. + */ +@Category({MasterTests.class, MediumTests.class}) +public class TestSnapshotFileCache { + + private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static long sequenceId = 0; + private static FileSystem fs; + private static Path rootDir; + + @BeforeClass + public static void startCluster() throws Exception { + UTIL.startMiniDFSCluster(1); + fs = UTIL.getDFSCluster().getFileSystem(); + rootDir = UTIL.getDefaultRootDirPath(); + } + + @AfterClass + public static void stopCluster() throws Exception { + UTIL.shutdownMiniDFSCluster(); + } + + @After + public void cleanupFiles() throws Exception { + // cleanup the snapshot directory + Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); + fs.delete(snapshotDir, true); + } + + @Test(timeout = 10000000) + public void testLoadAndDelete() throws IOException { + // don't refresh the cache unless we tell it to + long period = Long.MAX_VALUE; + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + + createAndTestSnapshotV1(cache, "snapshot1a", false, true); + createAndTestSnapshotV1(cache, "snapshot1b", true, true); + + createAndTestSnapshotV2(cache, "snapshot2a", false, true); + createAndTestSnapshotV2(cache, "snapshot2b", true, true); + } + + @Test + public void testReloadModifiedDirectory() throws IOException { + // don't refresh the cache unless we tell it to + long period = Long.MAX_VALUE; + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + + createAndTestSnapshotV1(cache, "snapshot1", false, true); + // now delete the snapshot and add a file with a different name + createAndTestSnapshotV1(cache, "snapshot1", false, false); + + createAndTestSnapshotV2(cache, "snapshot2", false, true); + // now delete the snapshot and add a file with a different name + createAndTestSnapshotV2(cache, "snapshot2", false, false); + } + + @Test + public void testSnapshotTempDirReload() throws IOException { + long period = Long.MAX_VALUE; + // This doesn't refresh cache until we invoke it explicitly + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + + // Add a new non-tmp snapshot + createAndTestSnapshotV1(cache, "snapshot0v1", false, false); + createAndTestSnapshotV1(cache, "snapshot0v2", false, false); + + // Add a new tmp snapshot + createAndTestSnapshotV2(cache, "snapshot1", true, false); + + // Add another tmp snapshot + createAndTestSnapshotV2(cache, "snapshot2", true, false); + } + + @Test + public void testWeNeverCacheTmpDirAndLoadIt() throws Exception { + + final AtomicInteger count = new AtomicInteger(0); + // don't refresh the cache unless we tell it to + long period = Long.MAX_VALUE; + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()) { + @Override + List getSnapshotsInProgress(final SnapshotManager snapshotManager) + throws IOException { + List result = super.getSnapshotsInProgress(snapshotManager); + count.incrementAndGet(); + return result; + } + + @Override public void triggerCacheRefreshForTesting() { + super.triggerCacheRefreshForTesting(); + } + }; + + SnapshotMock.SnapshotBuilder complete = + createAndTestSnapshotV1(cache, "snapshot", false, false); + + SnapshotMock.SnapshotBuilder inProgress = + createAndTestSnapshotV1(cache, "snapshotInProgress", true, false); + + int countBeforeCheck = count.get(); + + FSUtils.logFileSystemState(fs, rootDir, LOG); + + List allStoreFiles = getStoreFilesForSnapshot(complete); + Iterable deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); + assertTrue(Iterables.isEmpty(deletableFiles)); + // no need for tmp dir check as all files are accounted for. + assertEquals(0, count.get() - countBeforeCheck); + + + // add a random file to make sure we refresh + FileStatus randomFile = mockStoreFile(UUID.randomUUID().toString()); + allStoreFiles.add(randomFile); + deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); + assertEquals(randomFile, Iterables.getOnlyElement(deletableFiles)); + assertEquals(1, count.get() - countBeforeCheck); // we check the tmp directory + } + + private List getStoreFilesForSnapshot(SnapshotMock.SnapshotBuilder builder) + throws IOException { + final List allStoreFiles = Lists.newArrayList(); + SnapshotReferenceUtil + .visitReferencedFiles(UTIL.getConfiguration(), fs, builder.getSnapshotsDir(), + new SnapshotReferenceUtil.SnapshotVisitor() { + @Override public void storeFile(HRegionInfo regionInfo, String familyName, + SnapshotProtos.SnapshotRegionManifest.StoreFile storeFile) throws IOException { + FileStatus status = mockStoreFile(storeFile.getName()); + allStoreFiles.add(status); + } + }); + return allStoreFiles; + } + + private FileStatus mockStoreFile(String storeFileName) { + FileStatus status = mock(FileStatus.class); + Path path = mock(Path.class); + when(path.getName()).thenReturn(storeFileName); + when(status.getPath()).thenReturn(path); + return status; + } + + class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { + public Collection filesUnderSnapshot(final Path snapshotDir) throws IOException { + Collection files = new HashSet(); + files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir)); + return files; + } + }; + + private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache, + final String name, final boolean tmp, final boolean removeOnExit) throws IOException { + SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); + SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name); + createAndTestSnapshot(cache, builder, tmp, removeOnExit); + return builder; + } + + private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, + final boolean tmp, final boolean removeOnExit) throws IOException { + SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); + SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name); + createAndTestSnapshot(cache, builder, tmp, removeOnExit); + } + + private void createAndTestSnapshot(final SnapshotFileCache cache, + final SnapshotMock.SnapshotBuilder builder, + final boolean tmp, final boolean removeOnExit) throws IOException { + List files = new ArrayList(); + for (int i = 0; i < 3; ++i) { + for (Path filePath: builder.addRegion()) { + String fileName = filePath.getName(); + if (tmp) { + // We should be able to find all the files while the snapshot creation is in-progress + FSUtils.logFileSystemState(fs, rootDir, LOG); + Iterable nonSnapshot = getNonSnapshotFiles(cache, filePath); + assertFalse("Cache didn't find " + fileName, Iterables.contains(nonSnapshot, fileName)); + } + files.add(filePath); + } + } + + // Finalize the snapshot + if (!tmp) { + builder.commit(); + } + + // Make sure that all files are still present + for (Path path: files) { + Iterable nonSnapshotFiles = getNonSnapshotFiles(cache, path); + assertFalse("Cache didn't find " + path.getName(), + Iterables.contains(nonSnapshotFiles, path.getName())); + } + + FSUtils.logFileSystemState(fs, rootDir, LOG); + if (removeOnExit) { + LOG.debug("Deleting snapshot."); + fs.delete(builder.getSnapshotsDir(), true); + FSUtils.logFileSystemState(fs, rootDir, LOG); + + // The files should be in cache until next refresh + for (Path filePath: files) { + Iterable nonSnapshotFiles = getNonSnapshotFiles(cache, filePath); + assertFalse("Cache didn't find " + filePath.getName(), Iterables.contains(nonSnapshotFiles, + filePath.getName())); + } + + // then trigger a refresh + cache.triggerCacheRefreshForTesting(); + // and not it shouldn't find those files + for (Path filePath: files) { + Iterable nonSnapshotFiles = getNonSnapshotFiles(cache, filePath); + assertTrue("Cache found '" + filePath.getName() + "', but it shouldn't have.", + !Iterables.contains(nonSnapshotFiles, filePath.getName())); + } + } + } + + private Iterable getNonSnapshotFiles(SnapshotFileCache cache, Path storeFile) + throws IOException { + return cache.getUnreferencedFiles( + Arrays.asList(FSUtils.listStatus(fs, storeFile.getParent())), null + ); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotHFileCleaner.java new file mode 100644 index 0000000..98cd136 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/snapshot/TestSnapshotHFileCleaner.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.fs.legacy.snapshot; + +import static org.junit.Assert.assertFalse; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot + */ +@Category({MasterTests.class, SmallTests.class}) +public class TestSnapshotHFileCleaner { + + private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String TABLE_NAME_STR = "testSnapshotManifest"; + private static final String SNAPSHOT_NAME_STR = "testSnapshotManifest-snapshot"; + private static Path rootDir; + private static FileSystem fs; + + /** + * Setup the test environment + */ + @BeforeClass + public static void setup() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + rootDir = FSUtils.getRootDir(conf); + fs = FileSystem.get(conf); + } + + + @AfterClass + public static void cleanup() throws IOException { + // cleanup + fs.delete(rootDir, true); + } + + @Test + public void testFindsSnapshotFilesWhenCleaning() throws IOException { + Configuration conf = TEST_UTIL.getConfiguration(); + FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); + Path rootDir = FSUtils.getRootDir(conf); + Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); + + FileSystem fs = FileSystem.get(conf); + SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner(); + cleaner.setConf(conf); + + // write an hfile to the snapshot directory + String snapshotName = "snapshot"; + byte[] snapshot = Bytes.toBytes(snapshotName); + TableName tableName = TableName.valueOf("table"); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + HRegionInfo mockRegion = new HRegionInfo(tableName); + Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName()); + Path familyDir = new Path(regionSnapshotDir, "family"); + // create a reference to a supposedly valid hfile + String hfile = "fd1e73e8a96c486090c5cec07b4894c4"; + Path refFile = new Path(familyDir, hfile); + + // make sure the reference file exists + fs.create(refFile); + + // create the hfile in the archive + fs.mkdirs(archivedHfileDir); + fs.createNewFile(new Path(archivedHfileDir, hfile)); + + // make sure that the file isn't deletable + assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile))); + } + + class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { + public Collection filesUnderSnapshot(final Path snapshotDir) throws IOException { + Collection files = new HashSet(); + files.addAll(SnapshotReferenceUtil.getHFileNames(TEST_UTIL.getConfiguration(), fs, snapshotDir)); + return files; + } + } + + /** + * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException, + * instead of an IOException + */ + @Test + public void testCorruptedRegionManifest() throws IOException { + SnapshotTestingUtils.SnapshotMock + snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); + SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( + SNAPSHOT_NAME_STR, TABLE_NAME_STR); + builder.addRegionV2(); + builder.corruptOneRegionManifest(); + + long period = Long.MAX_VALUE; + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + try { + cache.getSnapshotsInProgress(null); + } catch (CorruptedSnapshotException cse) { + LOG.info("Expected exception " + cse); + } finally { + fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); + } + } + + /** + * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException, + * instead of an IOException + */ + @Test + public void testCorruptedDataManifest() throws IOException { + SnapshotTestingUtils.SnapshotMock + snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); + SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( + SNAPSHOT_NAME_STR, TABLE_NAME_STR); + builder.addRegionV2(); + // consolidate to generate a data.manifest file + builder.consolidate(); + builder.corruptDataManifest(); + + long period = Long.MAX_VALUE; + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + try { + cache.getSnapshotsInProgress(null); + } catch (CorruptedSnapshotException cse) { + LOG.info("Expected exception " + cse); + } finally { + fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); + } + } + + /** + * HBASE-16464 + */ + @Test + public void testMissedTmpSnapshot() throws IOException { + SnapshotTestingUtils.SnapshotMock + snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); + SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( + SNAPSHOT_NAME_STR, TABLE_NAME_STR); + builder.addRegionV2(); + builder.missOneRegionSnapshotFile(); + + long period = Long.MAX_VALUE; + SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, + "test-snapshot-file-cache-refresh", new SnapshotFiles()); + cache.getSnapshotsInProgress(null); + assertFalse(fs.exists(builder.getSnapshotsDir())); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java deleted file mode 100644 index 92c7bb6..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java +++ /dev/null @@ -1,319 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.Stoppable; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.StoppableImplementation; -import org.junit.After; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -@Category({MasterTests.class, SmallTests.class}) -public class TestCleanerChore { - - private static final Log LOG = LogFactory.getLog(TestCleanerChore.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - - @After - public void cleanup() throws Exception { - // delete and recreate the test directory, ensuring a clean test dir between tests - UTIL.cleanupTestDir(); -} - - - @Test - public void testSavesFilesOnRequest() throws Exception { - Stoppable stop = new StoppableImplementation(); - Configuration conf = UTIL.getConfiguration(); - Path testDir = UTIL.getDataTestDir(); - FileSystem fs = UTIL.getTestFileSystem(); - String confKey = "hbase.test.cleaner.delegates"; - conf.set(confKey, NeverDelete.class.getName()); - - AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); - - // create the directory layout in the directory to clean - Path parent = new Path(testDir, "parent"); - Path file = new Path(parent, "someFile"); - fs.mkdirs(parent); - // touch a new file - fs.create(file).close(); - assertTrue("Test file didn't get created.", fs.exists(file)); - - // run the chore - chore.chore(); - - // verify all the files got deleted - assertTrue("File didn't get deleted", fs.exists(file)); - assertTrue("Empty directory didn't get deleted", fs.exists(parent)); - } - - @Test - public void testDeletesEmptyDirectories() throws Exception { - Stoppable stop = new StoppableImplementation(); - Configuration conf = UTIL.getConfiguration(); - Path testDir = UTIL.getDataTestDir(); - FileSystem fs = UTIL.getTestFileSystem(); - String confKey = "hbase.test.cleaner.delegates"; - conf.set(confKey, AlwaysDelete.class.getName()); - - AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); - - // create the directory layout in the directory to clean - Path parent = new Path(testDir, "parent"); - Path child = new Path(parent, "child"); - Path emptyChild = new Path(parent, "emptyChild"); - Path file = new Path(child, "someFile"); - fs.mkdirs(child); - fs.mkdirs(emptyChild); - // touch a new file - fs.create(file).close(); - // also create a file in the top level directory - Path topFile = new Path(testDir, "topFile"); - fs.create(topFile).close(); - assertTrue("Test file didn't get created.", fs.exists(file)); - assertTrue("Test file didn't get created.", fs.exists(topFile)); - - // run the chore - chore.chore(); - - // verify all the files got deleted - assertFalse("File didn't get deleted", fs.exists(topFile)); - assertFalse("File didn't get deleted", fs.exists(file)); - assertFalse("Empty directory didn't get deleted", fs.exists(child)); - assertFalse("Empty directory didn't get deleted", fs.exists(parent)); - } - - /** - * Test to make sure that we don't attempt to ask the delegate whether or not we should preserve a - * directory. - * @throws Exception on failure - */ - @Test - public void testDoesNotCheckDirectories() throws Exception { - Stoppable stop = new StoppableImplementation(); - Configuration conf = UTIL.getConfiguration(); - Path testDir = UTIL.getDataTestDir(); - FileSystem fs = UTIL.getTestFileSystem(); - String confKey = "hbase.test.cleaner.delegates"; - conf.set(confKey, AlwaysDelete.class.getName()); - - AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); - // spy on the delegate to ensure that we don't check for directories - AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); - AlwaysDelete spy = Mockito.spy(delegate); - chore.cleanersChain.set(0, spy); - - // create the directory layout in the directory to clean - Path parent = new Path(testDir, "parent"); - Path file = new Path(parent, "someFile"); - fs.mkdirs(parent); - assertTrue("Test parent didn't get created.", fs.exists(parent)); - // touch a new file - fs.create(file).close(); - assertTrue("Test file didn't get created.", fs.exists(file)); - - FileStatus fStat = fs.getFileStatus(parent); - chore.chore(); - // make sure we never checked the directory - Mockito.verify(spy, Mockito.never()).isFileDeletable(fStat); - Mockito.reset(spy); - } - - @Test - public void testStoppedCleanerDoesNotDeleteFiles() throws Exception { - Stoppable stop = new StoppableImplementation(); - Configuration conf = UTIL.getConfiguration(); - Path testDir = UTIL.getDataTestDir(); - FileSystem fs = UTIL.getTestFileSystem(); - String confKey = "hbase.test.cleaner.delegates"; - conf.set(confKey, AlwaysDelete.class.getName()); - - AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); - - // also create a file in the top level directory - Path topFile = new Path(testDir, "topFile"); - fs.create(topFile).close(); - assertTrue("Test file didn't get created.", fs.exists(topFile)); - - // stop the chore - stop.stop("testing stop"); - - // run the chore - chore.chore(); - - // test that the file still exists - assertTrue("File got deleted while chore was stopped", fs.exists(topFile)); - } - - /** - * While cleaning a directory, all the files in the directory may be deleted, but there may be - * another file added, in which case the directory shouldn't be deleted. - * @throws IOException on failure - */ - @Test - public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException { - Stoppable stop = new StoppableImplementation(); - Configuration conf = UTIL.getConfiguration(); - final Path testDir = UTIL.getDataTestDir(); - final FileSystem fs = UTIL.getTestFileSystem(); - String confKey = "hbase.test.cleaner.delegates"; - conf.set(confKey, AlwaysDelete.class.getName()); - - AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); - // spy on the delegate to ensure that we don't check for directories - AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); - AlwaysDelete spy = Mockito.spy(delegate); - chore.cleanersChain.set(0, spy); - - // create the directory layout in the directory to clean - final Path parent = new Path(testDir, "parent"); - Path file = new Path(parent, "someFile"); - fs.mkdirs(parent); - // touch a new file - fs.create(file).close(); - assertTrue("Test file didn't get created.", fs.exists(file)); - final Path addedFile = new Path(parent, "addedFile"); - - // when we attempt to delete the original file, add another file in the same directory - Mockito.doAnswer(new Answer() { - @Override - public Boolean answer(InvocationOnMock invocation) throws Throwable { - fs.create(addedFile).close(); - FSUtils.logFileSystemState(fs, testDir, LOG); - return (Boolean) invocation.callRealMethod(); - } - }).when(spy).isFileDeletable(Mockito.any(FileStatus.class)); - - // run the chore - chore.chore(); - - // make sure all the directories + added file exist, but the original file is deleted - assertTrue("Added file unexpectedly deleted", fs.exists(addedFile)); - assertTrue("Parent directory deleted unexpectedly", fs.exists(parent)); - assertFalse("Original file unexpectedly retained", fs.exists(file)); - Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class)); - Mockito.reset(spy); - } - - /** - * The cleaner runs in a loop, where it first checks to see all the files under a directory can be - * deleted. If they all can, then we try to delete the directory. However, a file may be added - * that directory to after the original check. This ensures that we don't accidentally delete that - * directory on and don't get spurious IOExceptions. - *

- * This was from HBASE-7465. - * @throws Exception on failure - */ - @Test - public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception { - Stoppable stop = new StoppableImplementation(); - // need to use a localutil to not break the rest of the test that runs on the local FS, which - // gets hosed when we start to use a minicluster. - HBaseTestingUtility localUtil = new HBaseTestingUtility(); - Configuration conf = localUtil.getConfiguration(); - final Path testDir = UTIL.getDataTestDir(); - final FileSystem fs = UTIL.getTestFileSystem(); - LOG.debug("Writing test data to: " + testDir); - String confKey = "hbase.test.cleaner.delegates"; - conf.set(confKey, AlwaysDelete.class.getName()); - - AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); - // spy on the delegate to ensure that we don't check for directories - AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0); - AlwaysDelete spy = Mockito.spy(delegate); - chore.cleanersChain.set(0, spy); - - // create the directory layout in the directory to clean - final Path parent = new Path(testDir, "parent"); - Path file = new Path(parent, "someFile"); - fs.mkdirs(parent); - // touch a new file - fs.create(file).close(); - assertTrue("Test file didn't get created.", fs.exists(file)); - final Path racyFile = new Path(parent, "addedFile"); - - // when we attempt to delete the original file, add another file in the same directory - Mockito.doAnswer(new Answer() { - @Override - public Boolean answer(InvocationOnMock invocation) throws Throwable { - fs.create(racyFile).close(); - FSUtils.logFileSystemState(fs, testDir, LOG); - return (Boolean) invocation.callRealMethod(); - } - }).when(spy).isFileDeletable(Mockito.any(FileStatus.class)); - - // attempt to delete the directory, which - if (chore.checkAndDeleteDirectory(parent)) { - throw new Exception( - "Reported success deleting directory, should have failed when adding file mid-iteration"); - } - - // make sure all the directories + added file exist, but the original file is deleted - assertTrue("Added file unexpectedly deleted", fs.exists(racyFile)); - assertTrue("Parent directory deleted unexpectedly", fs.exists(parent)); - assertFalse("Original file unexpectedly retained", fs.exists(file)); - Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class)); - } - - private static class AllValidPaths extends CleanerChore { - - public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs, - Path oldFileDir, String confkey) { - super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey); - } - - // all paths are valid - @Override - protected boolean validate(Path file) { - return true; - } - }; - - public static class AlwaysDelete extends BaseHFileCleanerDelegate { - @Override - public boolean isFileDeletable(FileStatus fStat) { - return true; - } - } - - public static class NeverDelete extends BaseHFileCleanerDelegate { - @Override - public boolean isFileDeletable(FileStatus fStat) { - return false; - } - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java deleted file mode 100644 index 6049701..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java +++ /dev/null @@ -1,263 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.ChoreService; -import org.apache.hadoop.hbase.CoordinatedStateManager; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.client.ClusterConnection; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.EnvironmentEdge; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({MasterTests.class, MediumTests.class}) -public class TestHFileCleaner { - private static final Log LOG = LogFactory.getLog(TestHFileCleaner.class); - - private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); - - @BeforeClass - public static void setupCluster() throws Exception { - // have to use a minidfs cluster because the localfs doesn't modify file times correctly - UTIL.startMiniDFSCluster(1); - } - - @AfterClass - public static void shutdownCluster() throws IOException { - UTIL.shutdownMiniDFSCluster(); - } - - @Test - public void testTTLCleaner() throws IOException, InterruptedException { - FileSystem fs = UTIL.getDFSCluster().getFileSystem(); - Path root = UTIL.getDataTestDirOnTestFS(); - Path file = new Path(root, "file"); - fs.createNewFile(file); - long createTime = System.currentTimeMillis(); - assertTrue("Test file not created!", fs.exists(file)); - TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner(); - // update the time info for the file, so the cleaner removes it - fs.setTimes(file, createTime - 100, -1); - Configuration conf = UTIL.getConfiguration(); - conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100); - cleaner.setConf(conf); - assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs) - + " with create time:" + createTime, cleaner.isFileDeletable(fs.getFileStatus(file))); - } - - /** - * @param file to check - * @return loggable information about the file - */ - private String getFileStats(Path file, FileSystem fs) throws IOException { - FileStatus status = fs.getFileStatus(file); - return "File" + file + ", mtime:" + status.getModificationTime() + ", atime:" - + status.getAccessTime(); - } - - @Test(timeout = 60 *1000) - public void testHFileCleaning() throws Exception { - final EnvironmentEdge originalEdge = EnvironmentEdgeManager.getDelegate(); - String prefix = "someHFileThatWouldBeAUUID"; - Configuration conf = UTIL.getConfiguration(); - // set TTL - long ttl = 2000; - conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, - "org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner"); - conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); - Server server = new DummyServer(); - Path archivedHfileDir = new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY); - FileSystem fs = FileSystem.get(conf); - HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir); - - // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files - final long createTime = System.currentTimeMillis(); - fs.delete(archivedHfileDir, true); - fs.mkdirs(archivedHfileDir); - // Case 1: 1 invalid file, which should be deleted directly - fs.createNewFile(new Path(archivedHfileDir, "dfd-dfd")); - // Case 2: 1 "recent" file, not even deletable for the first log cleaner - // (TimeToLiveLogCleaner), so we are not going down the chain - LOG.debug("Now is: " + createTime); - for (int i = 1; i < 32; i++) { - // Case 3: old files which would be deletable for the first log cleaner - // (TimeToLiveHFileCleaner), - Path fileName = new Path(archivedHfileDir, (prefix + "." + (createTime + i))); - fs.createNewFile(fileName); - // set the creation time past ttl to ensure that it gets removed - fs.setTimes(fileName, createTime - ttl - 1, -1); - LOG.debug("Creating " + getFileStats(fileName, fs)); - } - - // Case 2: 1 newer file, not even deletable for the first log cleaner - // (TimeToLiveLogCleaner), so we are not going down the chain - Path saved = new Path(archivedHfileDir, prefix + ".00000000000"); - fs.createNewFile(saved); - // set creation time within the ttl - fs.setTimes(saved, createTime - ttl / 2, -1); - LOG.debug("Creating " + getFileStats(saved, fs)); - for (FileStatus stat : fs.listStatus(archivedHfileDir)) { - LOG.debug(stat.getPath().toString()); - } - - assertEquals(33, fs.listStatus(archivedHfileDir).length); - - // set a custom edge manager to handle time checking - EnvironmentEdge setTime = new EnvironmentEdge() { - @Override - public long currentTime() { - return createTime; - } - }; - EnvironmentEdgeManager.injectEdge(setTime); - - // run the chore - cleaner.chore(); - - // ensure we only end up with the saved file - assertEquals(1, fs.listStatus(archivedHfileDir).length); - - for (FileStatus file : fs.listStatus(archivedHfileDir)) { - LOG.debug("Kept hfiles: " + file.getPath().getName()); - } - - // reset the edge back to the original edge - EnvironmentEdgeManager.injectEdge(originalEdge); - } - - @Test - public void testRemovesEmptyDirectories() throws Exception { - Configuration conf = UTIL.getConfiguration(); - // no cleaner policies = delete all files - conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); - Server server = new DummyServer(); - Path archivedHfileDir = new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY); - - // setup the cleaner - FileSystem fs = UTIL.getDFSCluster().getFileSystem(); - HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir); - - // make all the directories for archiving files - Path table = new Path(archivedHfileDir, "table"); - Path region = new Path(table, "regionsomthing"); - Path family = new Path(region, "fam"); - Path file = new Path(family, "file12345"); - fs.mkdirs(family); - if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family); - fs.create(file).close(); - if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file); - - // run the chore to cleanup the files (and the directories above it) - cleaner.chore(); - - // make sure all the parent directories get removed - assertFalse("family directory not removed for empty directory", fs.exists(family)); - assertFalse("region directory not removed for empty directory", fs.exists(region)); - assertFalse("table directory not removed for empty directory", fs.exists(table)); - assertTrue("archive directory", fs.exists(archivedHfileDir)); - } - - static class DummyServer implements Server { - - @Override - public Configuration getConfiguration() { - return UTIL.getConfiguration(); - } - - @Override - public ZooKeeperWatcher getZooKeeper() { - try { - return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); - } catch (IOException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public CoordinatedStateManager getCoordinatedStateManager() { - return null; - } - - @Override - public ClusterConnection getConnection() { - return null; - } - - @Override - public MetaTableLocator getMetaTableLocator() { - return null; - } - - @Override - public ServerName getServerName() { - return ServerName.valueOf("regionserver,60020,000000"); - } - - @Override - public void abort(String why, Throwable e) { - } - - @Override - public boolean isAborted() { - return false; - } - - @Override - public void stop(String why) { - } - - @Override - public boolean isStopped() { - return false; - } - - @Override - public ChoreService getChoreService() { - return null; - } - - @Override - public ClusterConnection getClusterConnection() { - // TODO Auto-generated method stub - return null; - } - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java deleted file mode 100644 index 0401ae8..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java +++ /dev/null @@ -1,201 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CategoryBasedTimeout; -import org.apache.hadoop.hbase.ChoreService; -import org.apache.hadoop.hbase.CoordinatedStateManager; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.ClusterConnection; -import org.apache.hadoop.hbase.io.HFileLink; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.HFileArchiveUtil; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; - -/** - * Test the HFileLink Cleaner. - * HFiles with links cannot be deleted until a link is present. - */ -@Category({MasterTests.class, MediumTests.class}) -public class TestHFileLinkCleaner { - @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). - withLookingForStuckThread(true).build(); - - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - @Test - public void testHFileLinkCleaning() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); - conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName()); - Path rootDir = FSUtils.getRootDir(conf); - FileSystem fs = FileSystem.get(conf); - - final TableName tableName = TableName.valueOf("test-table"); - final TableName tableLinkName = TableName.valueOf("test-link"); - final String hfileName = "1234567890"; - final String familyName = "cf"; - - HRegionInfo hri = new HRegionInfo(tableName); - HRegionInfo hriLink = new HRegionInfo(tableLinkName); - - Path archiveDir = HFileArchiveUtil.getArchivePath(conf); - Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, - tableName, hri.getEncodedName(), familyName); - Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, - tableLinkName, hriLink.getEncodedName(), familyName); - - // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf); - Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName); - fs.mkdirs(familyPath); - Path hfilePath = new Path(familyPath, hfileName); - fs.createNewFile(hfilePath); - - // Create link to hfile - Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName, - hriLink.getEncodedName(), familyName); - fs.mkdirs(familyLinkPath); - HFileLink.create(conf, fs, familyLinkPath, hri, hfileName); - Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); - assertTrue(fs.exists(linkBackRefDir)); - FileStatus[] backRefs = fs.listStatus(linkBackRefDir); - assertEquals(1, backRefs.length); - Path linkBackRef = backRefs[0].getPath(); - - // Initialize cleaner - final long ttl = 1000; - conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); - Server server = new DummyServer(); - HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir); - - // Link backref cannot be removed - cleaner.chore(); - assertTrue(fs.exists(linkBackRef)); - assertTrue(fs.exists(hfilePath)); - - // Link backref can be removed - fs.rename(FSUtils.getTableDir(rootDir, tableLinkName), - FSUtils.getTableDir(archiveDir, tableLinkName)); - cleaner.chore(); - assertFalse("Link should be deleted", fs.exists(linkBackRef)); - - // HFile can be removed - Thread.sleep(ttl * 2); - cleaner.chore(); - assertFalse("HFile should be deleted", fs.exists(hfilePath)); - - // Remove everything - for (int i = 0; i < 4; ++i) { - Thread.sleep(ttl * 2); - cleaner.chore(); - } - assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName))); - assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName))); - } - - private static Path getFamilyDirPath (final Path rootDir, final TableName table, - final String region, final String family) { - return new Path(new Path(FSUtils.getTableDir(rootDir, table), region), family); - } - - static class DummyServer implements Server { - - @Override - public Configuration getConfiguration() { - return TEST_UTIL.getConfiguration(); - } - - @Override - public ZooKeeperWatcher getZooKeeper() { - try { - return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); - } catch (IOException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public CoordinatedStateManager getCoordinatedStateManager() { - return null; - } - - @Override - public ClusterConnection getConnection() { - return null; - } - - @Override - public MetaTableLocator getMetaTableLocator() { - return null; - } - - @Override - public ServerName getServerName() { - return ServerName.valueOf("regionserver,60020,000000"); - } - - @Override - public void abort(String why, Throwable e) {} - - @Override - public boolean isAborted() { - return false; - } - - @Override - public void stop(String why) {} - - @Override - public boolean isStopped() { - return false; - } - - @Override - public ChoreService getChoreService() { - return null; - } - - @Override - public ClusterConnection getClusterConnection() { - // TODO Auto-generated method stub - return null; - } - } -} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java deleted file mode 100644 index b6b5492..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java +++ /dev/null @@ -1,309 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.spy; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.net.URLEncoder; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - -import com.google.common.collect.Lists; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Abortable; -import org.apache.hadoop.hbase.ChoreService; -import org.apache.hadoop.hbase.CoordinatedStateManager; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.client.ClusterConnection; -import org.apache.hadoop.hbase.replication.ReplicationFactory; -import org.apache.hadoop.hbase.replication.ReplicationQueues; -import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments; -import org.apache.hadoop.hbase.replication.ReplicationQueuesClient; -import org.apache.hadoop.hbase.replication.ReplicationQueuesClientZKImpl; -import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner; -import org.apache.hadoop.hbase.replication.regionserver.Replication; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.data.Stat; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.mockito.Mockito; - -@Category({MasterTests.class, MediumTests.class}) -public class TestLogsCleaner { - - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - TEST_UTIL.startMiniZKCluster(); - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniZKCluster(); - } - - @Test - public void testLogCleaning() throws Exception{ - Configuration conf = TEST_UTIL.getConfiguration(); - // set TTL - long ttl = 10000; - conf.setLong("hbase.master.logcleaner.ttl", ttl); - Replication.decorateMasterConfiguration(conf); - Server server = new DummyServer(); - ReplicationQueues repQueues = - ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, server, server.getZooKeeper())); - repQueues.init(server.getServerName().toString()); - final Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(), - HConstants.HREGION_OLDLOGDIR_NAME); - String fakeMachineName = - URLEncoder.encode(server.getServerName().toString(), "UTF8"); - - final FileSystem fs = FileSystem.get(conf); - - // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files - long now = System.currentTimeMillis(); - fs.delete(oldLogDir, true); - fs.mkdirs(oldLogDir); - // Case 1: 2 invalid files, which would be deleted directly - fs.createNewFile(new Path(oldLogDir, "a")); - fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a")); - // Case 2: 1 "recent" file, not even deletable for the first log cleaner - // (TimeToLiveLogCleaner), so we are not going down the chain - System.out.println("Now is: " + now); - for (int i = 1; i < 31; i++) { - // Case 3: old files which would be deletable for the first log cleaner - // (TimeToLiveLogCleaner), and also for the second (ReplicationLogCleaner) - Path fileName = new Path(oldLogDir, fakeMachineName + "." + (now - i) ); - fs.createNewFile(fileName); - // Case 4: put 3 old log files in ZK indicating that they are scheduled - // for replication so these files would pass the first log cleaner - // (TimeToLiveLogCleaner) but would be rejected by the second - // (ReplicationLogCleaner) - if (i % (30/3) == 1) { - repQueues.addLog(fakeMachineName, fileName.getName()); - System.out.println("Replication log file: " + fileName); - } - } - - // sleep for sometime to get newer modifcation time - Thread.sleep(ttl); - fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now)); - - // Case 2: 1 newer file, not even deletable for the first log cleaner - // (TimeToLiveLogCleaner), so we are not going down the chain - fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000) )); - - for (FileStatus stat : fs.listStatus(oldLogDir)) { - System.out.println(stat.getPath().toString()); - } - - assertEquals(34, fs.listStatus(oldLogDir).length); - - LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir); - - cleaner.chore(); - - // We end up with the current log file, a newer one and the 3 old log - // files which are scheduled for replication - TEST_UTIL.waitFor(1000, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return 5 == fs.listStatus(oldLogDir).length; - } - }); - - for (FileStatus file : fs.listStatus(oldLogDir)) { - System.out.println("Kept log files: " + file.getPath().getName()); - } - } - - @Test(timeout=5000) - public void testZnodeCversionChange() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - ReplicationLogCleaner cleaner = new ReplicationLogCleaner(); - cleaner.setConf(conf); - - ReplicationQueuesClientZKImpl rqcMock = Mockito.mock(ReplicationQueuesClientZKImpl.class); - Mockito.when(rqcMock.getQueuesZNodeCversion()).thenReturn(1, 2, 3, 4); - - Field rqc = ReplicationLogCleaner.class.getDeclaredField("replicationQueues"); - rqc.setAccessible(true); - - rqc.set(cleaner, rqcMock); - - // This should return eventually when cversion stabilizes - cleaner.getDeletableFiles(new LinkedList()); - } - - /** - * ReplicationLogCleaner should be able to ride over ZooKeeper errors without - * aborting. - */ - @Test - public void testZooKeeperAbort() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - ReplicationLogCleaner cleaner = new ReplicationLogCleaner(); - - List dummyFiles = Lists.newArrayList( - new FileStatus(100, false, 3, 100, System.currentTimeMillis(), new Path("log1")), - new FileStatus(100, false, 3, 100, System.currentTimeMillis(), new Path("log2")) - ); - - FaultyZooKeeperWatcher faultyZK = - new FaultyZooKeeperWatcher(conf, "testZooKeeperAbort-faulty", null); - try { - faultyZK.init(); - cleaner.setConf(conf, faultyZK); - // should keep all files due to a ConnectionLossException getting the queues znodes - Iterable toDelete = cleaner.getDeletableFiles(dummyFiles); - assertFalse(toDelete.iterator().hasNext()); - assertFalse(cleaner.isStopped()); - } finally { - faultyZK.close(); - } - - // when zk is working both files should be returned - cleaner = new ReplicationLogCleaner(); - ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testZooKeeperAbort-normal", null); - try { - cleaner.setConf(conf, zkw); - Iterable filesToDelete = cleaner.getDeletableFiles(dummyFiles); - Iterator iter = filesToDelete.iterator(); - assertTrue(iter.hasNext()); - assertEquals(new Path("log1"), iter.next().getPath()); - assertTrue(iter.hasNext()); - assertEquals(new Path("log2"), iter.next().getPath()); - assertFalse(iter.hasNext()); - } finally { - zkw.close(); - } - } - - static class DummyServer implements Server { - - @Override - public Configuration getConfiguration() { - return TEST_UTIL.getConfiguration(); - } - - @Override - public ZooKeeperWatcher getZooKeeper() { - try { - return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); - } catch (IOException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public CoordinatedStateManager getCoordinatedStateManager() { - return null; - } - - @Override - public ClusterConnection getConnection() { - return null; - } - - @Override - public MetaTableLocator getMetaTableLocator() { - return null; - } - - @Override - public ServerName getServerName() { - return ServerName.valueOf("regionserver,60020,000000"); - } - - @Override - public void abort(String why, Throwable e) {} - - @Override - public boolean isAborted() { - return false; - } - - @Override - public void stop(String why) {} - - @Override - public boolean isStopped() { - return false; - } - - @Override - public ChoreService getChoreService() { - return null; - } - - @Override - public ClusterConnection getClusterConnection() { - // TODO Auto-generated method stub - return null; - } - } - - static class FaultyZooKeeperWatcher extends ZooKeeperWatcher { - private RecoverableZooKeeper zk; - - public FaultyZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable) - throws ZooKeeperConnectionException, IOException { - super(conf, identifier, abortable); - } - - public void init() throws Exception { - this.zk = spy(super.getRecoverableZooKeeper()); - doThrow(new KeeperException.ConnectionLossException()) - .when(zk).getData("/hbase/replication/rs", null, new Stat()); - } - - public RecoverableZooKeeper getRecoverableZooKeeper() { - return zk; - } - } -} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java deleted file mode 100644 index fc3e516..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java +++ /dev/null @@ -1,341 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.spy; - -import com.google.common.collect.Lists; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Abortable; -import org.apache.hadoop.hbase.ChoreService; -import org.apache.hadoop.hbase.CoordinatedStateManager; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.client.ClusterConnection; -import org.apache.hadoop.hbase.replication.ReplicationException; -import org.apache.hadoop.hbase.replication.ReplicationFactory; -import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; -import org.apache.hadoop.hbase.replication.ReplicationPeers; -import org.apache.hadoop.hbase.replication.ReplicationQueues; -import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments; -import org.apache.hadoop.hbase.replication.ReplicationQueuesClient; -import org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl; -import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner; -import org.apache.hadoop.hbase.replication.regionserver.Replication; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.data.Stat; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.mockito.Mockito; - -@Category({ MasterTests.class, SmallTests.class }) -public class TestReplicationHFileCleaner { - private static final Log LOG = LogFactory.getLog(ReplicationQueuesZKImpl.class); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static Server server; - private static ReplicationQueues rq; - private static ReplicationPeers rp; - private static final String peerId = "TestReplicationHFileCleaner"; - private static Configuration conf = TEST_UTIL.getConfiguration(); - static FileSystem fs = null; - Path root; - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - TEST_UTIL.startMiniZKCluster(); - server = new DummyServer(); - conf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); - Replication.decorateMasterConfiguration(conf); - rp = ReplicationFactory.getReplicationPeers(server.getZooKeeper(), conf, server); - rp.init(); - rq = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, server, server.getZooKeeper())); - rq.init(server.getServerName().toString()); - try { - fs = FileSystem.get(conf); - } finally { - if (fs != null) { - fs.close(); - } - } - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniZKCluster(); - } - - @Before - public void setup() throws ReplicationException, IOException { - root = TEST_UTIL.getDataTestDirOnTestFS(); - rp.registerPeer(peerId, new ReplicationPeerConfig().setClusterKey(TEST_UTIL.getClusterKey())); - rq.addPeerToHFileRefs(peerId); - } - - @After - public void cleanup() throws ReplicationException { - try { - fs.delete(root, true); - } catch (IOException e) { - LOG.warn("Failed to delete files recursively from path " + root); - } - rp.unregisterPeer(peerId); - } - - @Test - public void testIsFileDeletable() throws IOException, ReplicationException { - // 1. Create a file - Path file = new Path(root, "testIsFileDeletableWithNoHFileRefs"); - fs.createNewFile(file); - // 2. Assert file is successfully created - assertTrue("Test file not created!", fs.exists(file)); - ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner(); - cleaner.setConf(conf); - // 3. Assert that file as is should be deletable - assertTrue("Cleaner should allow to delete this file as there is no hfile reference node " - + "for it in the queue.", - cleaner.isFileDeletable(fs.getFileStatus(file))); - - List files = new ArrayList(1); - files.add(file.getName()); - // 4. Add the file to hfile-refs queue - rq.addHFileRefs(peerId, files); - // 5. Assert file should not be deletable - assertFalse("Cleaner should not allow to delete this file as there is a hfile reference node " - + "for it in the queue.", - cleaner.isFileDeletable(fs.getFileStatus(file))); - } - - @Test - public void testGetDeletableFiles() throws Exception { - // 1. Create two files and assert that they do not exist - Path notDeletablefile = new Path(root, "testGetDeletableFiles_1"); - fs.createNewFile(notDeletablefile); - assertTrue("Test file not created!", fs.exists(notDeletablefile)); - Path deletablefile = new Path(root, "testGetDeletableFiles_2"); - fs.createNewFile(deletablefile); - assertTrue("Test file not created!", fs.exists(deletablefile)); - - List files = new ArrayList(2); - FileStatus f = new FileStatus(); - f.setPath(deletablefile); - files.add(f); - f = new FileStatus(); - f.setPath(notDeletablefile); - files.add(f); - - List hfiles = new ArrayList<>(1); - hfiles.add(notDeletablefile.getName()); - // 2. Add one file to hfile-refs queue - rq.addHFileRefs(peerId, hfiles); - - ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner(); - cleaner.setConf(conf); - Iterator deletableFilesIterator = cleaner.getDeletableFiles(files).iterator(); - int i = 0; - while (deletableFilesIterator.hasNext() && i < 2) { - i++; - } - // 5. Assert one file should not be deletable and it is present in the list returned - if (i > 2) { - fail("File " + notDeletablefile - + " should not be deletable as its hfile reference node is not added."); - } - assertTrue(deletableFilesIterator.next().getPath().equals(deletablefile)); - } - - /* - * Test for HBASE-14621. This test will not assert directly anything. Without the fix the test - * will end up in a infinite loop, so it will timeout. - */ - @Test(timeout = 15000) - public void testForDifferntHFileRefsZnodeVersion() throws Exception { - // 1. Create a file - Path file = new Path(root, "testForDifferntHFileRefsZnodeVersion"); - fs.createNewFile(file); - // 2. Assert file is successfully created - assertTrue("Test file not created!", fs.exists(file)); - ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner(); - cleaner.setConf(conf); - - ReplicationQueuesClient replicationQueuesClient = Mockito.mock(ReplicationQueuesClient.class); - //Return different znode version for each call - Mockito.when(replicationQueuesClient.getHFileRefsNodeChangeVersion()).thenReturn(1, 2); - - Class cleanerClass = cleaner.getClass(); - Field rqc = cleanerClass.getDeclaredField("rqc"); - rqc.setAccessible(true); - rqc.set(cleaner, replicationQueuesClient); - - cleaner.isFileDeletable(fs.getFileStatus(file)); - } - - /** - * ReplicationHFileCleaner should be able to ride over ZooKeeper errors without aborting. - */ - @Test - public void testZooKeeperAbort() throws Exception { - ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner(); - - List dummyFiles = - Lists.newArrayList(new FileStatus(100, false, 3, 100, System.currentTimeMillis(), new Path( - "hfile1")), new FileStatus(100, false, 3, 100, System.currentTimeMillis(), new Path( - "hfile2"))); - - FaultyZooKeeperWatcher faultyZK = - new FaultyZooKeeperWatcher(conf, "testZooKeeperAbort-faulty", null); - try { - faultyZK.init(); - cleaner.setConf(conf, faultyZK); - // should keep all files due to a ConnectionLossException getting the queues znodes - Iterable toDelete = cleaner.getDeletableFiles(dummyFiles); - assertFalse(toDelete.iterator().hasNext()); - assertFalse(cleaner.isStopped()); - } finally { - faultyZK.close(); - } - - // when zk is working both files should be returned - cleaner = new ReplicationHFileCleaner(); - ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testZooKeeperAbort-normal", null); - try { - cleaner.setConf(conf, zkw); - Iterable filesToDelete = cleaner.getDeletableFiles(dummyFiles); - Iterator iter = filesToDelete.iterator(); - assertTrue(iter.hasNext()); - assertEquals(new Path("hfile1"), iter.next().getPath()); - assertTrue(iter.hasNext()); - assertEquals(new Path("hfile2"), iter.next().getPath()); - assertFalse(iter.hasNext()); - } finally { - zkw.close(); - } - } - - static class DummyServer implements Server { - - @Override - public Configuration getConfiguration() { - return TEST_UTIL.getConfiguration(); - } - - @Override - public ZooKeeperWatcher getZooKeeper() { - try { - return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); - } catch (IOException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public CoordinatedStateManager getCoordinatedStateManager() { - return null; - } - - @Override - public ClusterConnection getConnection() { - return null; - } - - @Override - public MetaTableLocator getMetaTableLocator() { - return null; - } - - @Override - public ServerName getServerName() { - return ServerName.valueOf("regionserver,60020,000000"); - } - - @Override - public void abort(String why, Throwable e) { - } - - @Override - public boolean isAborted() { - return false; - } - - @Override - public void stop(String why) { - } - - @Override - public boolean isStopped() { - return false; - } - - @Override - public ChoreService getChoreService() { - return null; - } - - @Override - public ClusterConnection getClusterConnection() { - // TODO Auto-generated method stub - return null; - } - } - - static class FaultyZooKeeperWatcher extends ZooKeeperWatcher { - private RecoverableZooKeeper zk; - public FaultyZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable) - throws ZooKeeperConnectionException, IOException { - super(conf, identifier, abortable); - } - - public void init() throws Exception { - this.zk = spy(super.getRecoverableZooKeeper()); - doThrow(new KeeperException.ConnectionLossException()) - .when(zk).getData("/hbase/replication/hfile-refs", null, new Stat()); - } - - public RecoverableZooKeeper getRecoverableZooKeeper() { - return zk; - } - } -} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java deleted file mode 100644 index 6d0f8dd..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java +++ /dev/null @@ -1,404 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.cleaner; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.snapshot.DisabledTableSnapshotHandler; -import org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.regionserver.CompactedHFilesDischarger; -import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; -import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; -import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.mockito.Mockito; - -import com.google.common.collect.Lists; -import com.google.protobuf.ServiceException; - -/** - * Test the master-related aspects of a snapshot - */ -@Category({MasterTests.class, MediumTests.class}) -public class TestSnapshotFromMaster { - - private static final Log LOG = LogFactory.getLog(TestSnapshotFromMaster.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static final int NUM_RS = 2; - private static Path rootDir; - private static FileSystem fs; - private static HMaster master; - - // for hfile archiving test. - private static Path archiveDir; - private static final byte[] TEST_FAM = Bytes.toBytes("fam"); - private static final TableName TABLE_NAME = - TableName.valueOf("test"); - // refresh the cache every 1/2 second - private static final long cacheRefreshPeriod = 500; - private static final int blockingStoreFiles = 12; - - /** - * Setup the config for the cluster - */ - @BeforeClass - public static void setupCluster() throws Exception { - setupConf(UTIL.getConfiguration()); - UTIL.startMiniCluster(NUM_RS); - fs = UTIL.getDFSCluster().getFileSystem(); - master = UTIL.getMiniHBaseCluster().getMaster(); -// rootDir = master.getMasterStorage().getRootDir(); - archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); - } - - private static void setupConf(Configuration conf) { - // disable the ui - conf.setInt("hbase.regionsever.info.port", -1); - // change the flush size to a small amount, regulating number of store files - conf.setInt("hbase.hregion.memstore.flush.size", 25000); - // so make sure we get a compaction when doing a load, but keep around some - // files in the store - conf.setInt("hbase.hstore.compaction.min", 2); - conf.setInt("hbase.hstore.compactionThreshold", 5); - // block writes if we get to 12 store files - conf.setInt("hbase.hstore.blockingStoreFiles", blockingStoreFiles); - // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner) - conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, ""); - conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, ""); - // Enable snapshot - conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); - conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod); - conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, - ConstantSizeRegionSplitPolicy.class.getName()); - conf.setInt("hbase.hfile.compactions.cleaner.interval", 20 * 1000); - - } - - @Before - public void setup() throws Exception { - UTIL.createTable(TABLE_NAME, TEST_FAM); - master.getSnapshotManager().setSnapshotHandlerForTesting(TABLE_NAME, null); - } - - @After - public void tearDown() throws Exception { - UTIL.deleteTable(TABLE_NAME); - SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin()); - SnapshotTestingUtils.deleteArchiveDirectory(UTIL); - } - - @AfterClass - public static void cleanupTest() throws Exception { - try { - UTIL.shutdownMiniCluster(); - } catch (Exception e) { - // NOOP; - } - } - -// /** -// * Test that the contract from the master for checking on a snapshot are valid. -// *

-// *

    -// *
  1. If a snapshot fails with an error, we expect to get the source error.
  2. -// *
  3. If there is no snapshot name supplied, we should get an error.
  4. -// *
  5. If asking about a snapshot has hasn't occurred, you should get an error.
  6. -// *
-// */ -// @Test(timeout = 300000) -// public void testIsDoneContract() throws Exception { -// -// IsSnapshotDoneRequest.Builder builder = IsSnapshotDoneRequest.newBuilder(); -// -// String snapshotName = "asyncExpectedFailureTest"; -// -// // check that we get an exception when looking up snapshot where one hasn't happened -// SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), -// UnknownSnapshotException.class); -// -// // and that we get the same issue, even if we specify a name -// SnapshotDescription desc = SnapshotDescription.newBuilder() -// .setName(snapshotName).setTable(TABLE_NAME.getNameAsString()).build(); -// builder.setSnapshot(desc); -// SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), -// UnknownSnapshotException.class); -// -// // set a mock handler to simulate a snapshot -// DisabledTableSnapshotHandler mockHandler = Mockito.mock(DisabledTableSnapshotHandler.class); -// Mockito.when(mockHandler.getException()).thenReturn(null); -// Mockito.when(mockHandler.getSnapshot()).thenReturn(desc); -// Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true)); -// Mockito.when(mockHandler.getCompletionTimestamp()) -// .thenReturn(EnvironmentEdgeManager.currentTime()); -// -// master.getSnapshotManager() -// .setSnapshotHandlerForTesting(TABLE_NAME, mockHandler); -// -// // if we do a lookup without a snapshot name, we should fail - you should always know your name -// builder = IsSnapshotDoneRequest.newBuilder(); -// SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), -// UnknownSnapshotException.class); -// -// // then do the lookup for the snapshot that it is done -// builder.setSnapshot(desc); -// IsSnapshotDoneResponse response = -// master.getMasterRpcServices().isSnapshotDone(null, builder.build()); -// assertTrue("Snapshot didn't complete when it should have.", response.getDone()); -// -// // now try the case where we are looking for a snapshot we didn't take -// builder.setSnapshot(SnapshotDescription.newBuilder().setName("Not A Snapshot").build()); -// SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), -// UnknownSnapshotException.class); -// -// // then create a snapshot to the fs and make sure that we can find it when checking done -// snapshotName = "completed"; -// Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); -// desc = desc.toBuilder().setName(snapshotName).build(); -// SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshotDir, fs); -// -// builder.setSnapshot(desc); -// response = master.getMasterRpcServices().isSnapshotDone(null, builder.build()); -// assertTrue("Completed, on-disk snapshot not found", response.getDone()); -// } -// -// @Test(timeout = 300000) -// public void testGetCompletedSnapshots() throws Exception { -// // first check when there are no snapshots -// GetCompletedSnapshotsRequest request = GetCompletedSnapshotsRequest.newBuilder().build(); -// GetCompletedSnapshotsResponse response = -// master.getMasterRpcServices().getCompletedSnapshots(null, request); -// assertEquals("Found unexpected number of snapshots", 0, response.getSnapshotsCount()); -// -// // write one snapshot to the fs -// String snapshotName = "completed"; -// Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); -// SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); -// SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); -// -// // check that we get one snapshot -// response = master.getMasterRpcServices().getCompletedSnapshots(null, request); -// assertEquals("Found unexpected number of snapshots", 1, response.getSnapshotsCount()); -// List snapshots = response.getSnapshotsList(); -// List expected = Lists.newArrayList(snapshot); -// assertEquals("Returned snapshots don't match created snapshots", expected, snapshots); -// -// // write a second snapshot -// snapshotName = "completed_two"; -// snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); -// snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); -// SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); -// expected.add(snapshot); -// -// // check that we get one snapshot -// response = master.getMasterRpcServices().getCompletedSnapshots(null, request); -// assertEquals("Found unexpected number of snapshots", 2, response.getSnapshotsCount()); -// snapshots = response.getSnapshotsList(); -// assertEquals("Returned snapshots don't match created snapshots", expected, snapshots); -// } -// -// @Test(timeout = 300000) -// public void testDeleteSnapshot() throws Exception { -// -// String snapshotName = "completed"; -// SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build(); -// -// DeleteSnapshotRequest request = DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot) -// .build(); -// try { -// master.getMasterRpcServices().deleteSnapshot(null, request); -// fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist"); -// } catch (ServiceException e) { -// LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage()); -// } -// -// // write one snapshot to the fs -// Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); -// SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs); -// -// // then delete the existing snapshot,which shouldn't cause an exception to be thrown -// master.getMasterRpcServices().deleteSnapshot(null, request); -// } - - /** - * Test that the snapshot hfile archive cleaner works correctly. HFiles that are in snapshots - * should be retained, while those that are not in a snapshot should be deleted. - * @throws Exception on failure - */ - @Test(timeout = 300000) - public void testSnapshotHFileArchiving() throws Exception { - Admin admin = UTIL.getHBaseAdmin(); - // make sure we don't fail on listing snapshots - SnapshotTestingUtils.assertNoSnapshots(admin); - - // recreate test table with disabled compactions; otherwise compaction may happen before - // snapshot, the call after snapshot will be a no-op and checks will fail - UTIL.deleteTable(TABLE_NAME); - HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); - htd.setCompactionEnabled(false); - UTIL.createTable(htd, new byte[][] { TEST_FAM }, null); - - // load the table - for (int i = 0; i < blockingStoreFiles / 2; i ++) { - UTIL.loadTable(UTIL.getConnection().getTable(TABLE_NAME), TEST_FAM); - UTIL.flush(TABLE_NAME); - } - - // disable the table so we can take a snapshot - admin.disableTable(TABLE_NAME); - htd.setCompactionEnabled(true); - - // take a snapshot of the table - String snapshotName = "snapshot"; - byte[] snapshotNameBytes = Bytes.toBytes(snapshotName); - admin.snapshot(snapshotNameBytes, TABLE_NAME); - - LOG.info("After snapshot File-System state"); - FSUtils.logFileSystemState(fs, rootDir, LOG); - - // ensure we only have one snapshot - SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshotNameBytes, TABLE_NAME); - - // enable compactions now - admin.modifyTable(TABLE_NAME, htd); - - // renable the table so we can compact the regions - admin.enableTable(TABLE_NAME); - - // compact the files so we get some archived files for the table we just snapshotted - List regions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); - for (HRegion region : regions) { - region.waitForFlushesAndCompactions(); // enable can trigger a compaction, wait for it. - region.compactStores(); // min is 2 so will compact and archive - } - List regionServerThreads = UTIL.getMiniHBaseCluster() - .getRegionServerThreads(); - HRegionServer hrs = null; - for (RegionServerThread rs : regionServerThreads) { - if (!rs.getRegionServer().getOnlineRegions(TABLE_NAME).isEmpty()) { - hrs = rs.getRegionServer(); - break; - } - } - CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, hrs, false); - cleaner.chore(); - LOG.info("After compaction File-System state"); - FSUtils.logFileSystemState(fs, rootDir, LOG); - - // make sure the cleaner has run - LOG.debug("Running hfile cleaners"); - ensureHFileCleanersRun(); - LOG.info("After cleaners File-System state: " + rootDir); - FSUtils.logFileSystemState(fs, rootDir, LOG); - - // get the snapshot files for the table - Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); - Set snapshotHFiles = SnapshotReferenceUtil.getHFileNames( - UTIL.getConfiguration(), fs, snapshotTable); - // check that the files in the archive contain the ones that we need for the snapshot - LOG.debug("Have snapshot hfiles:"); - for (String fileName : snapshotHFiles) { - LOG.debug(fileName); - } - // get the archived files for the table - Collection archives = getHFiles(archiveDir, fs, TABLE_NAME); - - // get the hfiles for the table - Collection hfiles = getHFiles(rootDir, fs, TABLE_NAME); - - // and make sure that there is a proper subset - for (String fileName : snapshotHFiles) { - boolean exist = archives.contains(fileName) || hfiles.contains(fileName); - assertTrue("Archived hfiles " + archives - + " and table hfiles " + hfiles + " is missing snapshot file:" + fileName, exist); - } - - // delete the existing snapshot - admin.deleteSnapshot(snapshotNameBytes); - SnapshotTestingUtils.assertNoSnapshots(admin); - - // make sure that we don't keep around the hfiles that aren't in a snapshot - // make sure we wait long enough to refresh the snapshot hfile - List delegates = UTIL.getMiniHBaseCluster().getMaster() - .getHFileCleaner().cleanersChain; - for (BaseHFileCleanerDelegate delegate: delegates) { - if (delegate instanceof SnapshotHFileCleaner) { - ((SnapshotHFileCleaner)delegate).getFileCacheForTesting().triggerCacheRefreshForTesting(); - } - } - // run the cleaner again - LOG.debug("Running hfile cleaners"); - ensureHFileCleanersRun(); - LOG.info("After delete snapshot cleaners run File-System state"); - FSUtils.logFileSystemState(fs, rootDir, LOG); - - archives = getHFiles(archiveDir, fs, TABLE_NAME); - assertEquals("Still have some hfiles in the archive, when their snapshot has been deleted.", 0, - archives.size()); - } - - /** - * @return all the HFiles for a given table in the specified dir - * @throws IOException on expected failure - */ - private final Collection getHFiles(Path dir, FileSystem fs, TableName tableName) throws IOException { - Path tableDir = FSUtils.getTableDir(dir, tableName); - return SnapshotTestingUtils.listHFileNames(fs, tableDir); - } - - /** - * Make sure the {@link HFileCleaner HFileCleaners} run at least once - */ - private static void ensureHFileCleanersRun() { - UTIL.getHBaseCluster().getMaster().getHFileCleaner().chore(); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java deleted file mode 100644 index eb2c84c..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java +++ /dev/null @@ -1,282 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.snapshot; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; - -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; -import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; -import org.apache.hadoop.hbase.util.FSUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Test that we correctly reload the cache, filter directories, etc. - */ -@Category({MasterTests.class, MediumTests.class}) -public class TestSnapshotFileCache { - - private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static long sequenceId = 0; - private static FileSystem fs; - private static Path rootDir; - - @BeforeClass - public static void startCluster() throws Exception { - UTIL.startMiniDFSCluster(1); - fs = UTIL.getDFSCluster().getFileSystem(); - rootDir = UTIL.getDefaultRootDirPath(); - } - - @AfterClass - public static void stopCluster() throws Exception { - UTIL.shutdownMiniDFSCluster(); - } - - @After - public void cleanupFiles() throws Exception { - // cleanup the snapshot directory - Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); - fs.delete(snapshotDir, true); - } - - @Test(timeout = 10000000) - public void testLoadAndDelete() throws IOException { - // don't refresh the cache unless we tell it to - long period = Long.MAX_VALUE; - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, - "test-snapshot-file-cache-refresh", new SnapshotFiles()); - - createAndTestSnapshotV1(cache, "snapshot1a", false, true); - createAndTestSnapshotV1(cache, "snapshot1b", true, true); - - createAndTestSnapshotV2(cache, "snapshot2a", false, true); - createAndTestSnapshotV2(cache, "snapshot2b", true, true); - } - - @Test - public void testReloadModifiedDirectory() throws IOException { - // don't refresh the cache unless we tell it to - long period = Long.MAX_VALUE; - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, - "test-snapshot-file-cache-refresh", new SnapshotFiles()); - - createAndTestSnapshotV1(cache, "snapshot1", false, true); - // now delete the snapshot and add a file with a different name - createAndTestSnapshotV1(cache, "snapshot1", false, false); - - createAndTestSnapshotV2(cache, "snapshot2", false, true); - // now delete the snapshot and add a file with a different name - createAndTestSnapshotV2(cache, "snapshot2", false, false); - } - - @Test - public void testSnapshotTempDirReload() throws IOException { - long period = Long.MAX_VALUE; - // This doesn't refresh cache until we invoke it explicitly - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, - "test-snapshot-file-cache-refresh", new SnapshotFiles()); - - // Add a new non-tmp snapshot - createAndTestSnapshotV1(cache, "snapshot0v1", false, false); - createAndTestSnapshotV1(cache, "snapshot0v2", false, false); - - // Add a new tmp snapshot - createAndTestSnapshotV2(cache, "snapshot1", true, false); - - // Add another tmp snapshot - createAndTestSnapshotV2(cache, "snapshot2", true, false); - } - - @Test - public void testWeNeverCacheTmpDirAndLoadIt() throws Exception { - - final AtomicInteger count = new AtomicInteger(0); - // don't refresh the cache unless we tell it to - long period = Long.MAX_VALUE; - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, - "test-snapshot-file-cache-refresh", new SnapshotFiles()) { - @Override - List getSnapshotsInProgress(final SnapshotManager snapshotManager) - throws IOException { - List result = super.getSnapshotsInProgress(snapshotManager); - count.incrementAndGet(); - return result; - } - - @Override public void triggerCacheRefreshForTesting() { - super.triggerCacheRefreshForTesting(); - } - }; - - SnapshotMock.SnapshotBuilder complete = - createAndTestSnapshotV1(cache, "snapshot", false, false); - - SnapshotMock.SnapshotBuilder inProgress = - createAndTestSnapshotV1(cache, "snapshotInProgress", true, false); - - int countBeforeCheck = count.get(); - - FSUtils.logFileSystemState(fs, rootDir, LOG); - - List allStoreFiles = getStoreFilesForSnapshot(complete); - Iterable deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); - assertTrue(Iterables.isEmpty(deletableFiles)); - // no need for tmp dir check as all files are accounted for. - assertEquals(0, count.get() - countBeforeCheck); - - - // add a random file to make sure we refresh - FileStatus randomFile = mockStoreFile(UUID.randomUUID().toString()); - allStoreFiles.add(randomFile); - deletableFiles = cache.getUnreferencedFiles(allStoreFiles, null); - assertEquals(randomFile, Iterables.getOnlyElement(deletableFiles)); - assertEquals(1, count.get() - countBeforeCheck); // we check the tmp directory - } - - private List getStoreFilesForSnapshot(SnapshotMock.SnapshotBuilder builder) - throws IOException { - final List allStoreFiles = Lists.newArrayList(); - SnapshotReferenceUtil - .visitReferencedFiles(UTIL.getConfiguration(), fs, builder.getSnapshotsDir(), - new SnapshotReferenceUtil.SnapshotVisitor() { - @Override public void storeFile(HRegionInfo regionInfo, String familyName, - SnapshotProtos.SnapshotRegionManifest.StoreFile storeFile) throws IOException { - FileStatus status = mockStoreFile(storeFile.getName()); - allStoreFiles.add(status); - } - }); - return allStoreFiles; - } - - private FileStatus mockStoreFile(String storeFileName) { - FileStatus status = mock(FileStatus.class); - Path path = mock(Path.class); - when(path.getName()).thenReturn(storeFileName); - when(status.getPath()).thenReturn(path); - return status; - } - - class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { - public Collection filesUnderSnapshot(final Path snapshotDir) throws IOException { - Collection files = new HashSet(); - files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir)); - return files; - } - }; - - private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache, - final String name, final boolean tmp, final boolean removeOnExit) throws IOException { - SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); - SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name); - createAndTestSnapshot(cache, builder, tmp, removeOnExit); - return builder; - } - - private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, - final boolean tmp, final boolean removeOnExit) throws IOException { - SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); - SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name); - createAndTestSnapshot(cache, builder, tmp, removeOnExit); - } - - private void createAndTestSnapshot(final SnapshotFileCache cache, - final SnapshotMock.SnapshotBuilder builder, - final boolean tmp, final boolean removeOnExit) throws IOException { - List files = new ArrayList(); - for (int i = 0; i < 3; ++i) { - for (Path filePath: builder.addRegion()) { - String fileName = filePath.getName(); - if (tmp) { - // We should be able to find all the files while the snapshot creation is in-progress - FSUtils.logFileSystemState(fs, rootDir, LOG); - Iterable nonSnapshot = getNonSnapshotFiles(cache, filePath); - assertFalse("Cache didn't find " + fileName, Iterables.contains(nonSnapshot, fileName)); - } - files.add(filePath); - } - } - - // Finalize the snapshot - if (!tmp) { - builder.commit(); - } - - // Make sure that all files are still present - for (Path path: files) { - Iterable nonSnapshotFiles = getNonSnapshotFiles(cache, path); - assertFalse("Cache didn't find " + path.getName(), - Iterables.contains(nonSnapshotFiles, path.getName())); - } - - FSUtils.logFileSystemState(fs, rootDir, LOG); - if (removeOnExit) { - LOG.debug("Deleting snapshot."); - fs.delete(builder.getSnapshotsDir(), true); - FSUtils.logFileSystemState(fs, rootDir, LOG); - - // The files should be in cache until next refresh - for (Path filePath: files) { - Iterable nonSnapshotFiles = getNonSnapshotFiles(cache, filePath); - assertFalse("Cache didn't find " + filePath.getName(), Iterables.contains(nonSnapshotFiles, - filePath.getName())); - } - - // then trigger a refresh - cache.triggerCacheRefreshForTesting(); - // and not it shouldn't find those files - for (Path filePath: files) { - Iterable nonSnapshotFiles = getNonSnapshotFiles(cache, filePath); - assertTrue("Cache found '" + filePath.getName() + "', but it shouldn't have.", - !Iterables.contains(nonSnapshotFiles, filePath.getName())); - } - } - } - - private Iterable getNonSnapshotFiles(SnapshotFileCache cache, Path storeFile) - throws IOException { - return cache.getUnreferencedFiles( - Arrays.asList(FSUtils.listStatus(fs, storeFile.getParent())), null - ); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java deleted file mode 100644 index 1a27433..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.snapshot; - -import static org.junit.Assert.assertFalse; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; -import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot - */ -@Category({MasterTests.class, SmallTests.class}) -public class TestSnapshotHFileCleaner { - - private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static final String TABLE_NAME_STR = "testSnapshotManifest"; - private static final String SNAPSHOT_NAME_STR = "testSnapshotManifest-snapshot"; - private static Path rootDir; - private static FileSystem fs; - - /** - * Setup the test environment - */ - @BeforeClass - public static void setup() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - rootDir = FSUtils.getRootDir(conf); - fs = FileSystem.get(conf); - } - - - @AfterClass - public static void cleanup() throws IOException { - // cleanup - fs.delete(rootDir, true); - } - - @Test - public void testFindsSnapshotFilesWhenCleaning() throws IOException { - Configuration conf = TEST_UTIL.getConfiguration(); - FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); - Path rootDir = FSUtils.getRootDir(conf); - Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); - - FileSystem fs = FileSystem.get(conf); - SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner(); - cleaner.setConf(conf); - - // write an hfile to the snapshot directory - String snapshotName = "snapshot"; - byte[] snapshot = Bytes.toBytes(snapshotName); - TableName tableName = TableName.valueOf("table"); - Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); - HRegionInfo mockRegion = new HRegionInfo(tableName); - Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName()); - Path familyDir = new Path(regionSnapshotDir, "family"); - // create a reference to a supposedly valid hfile - String hfile = "fd1e73e8a96c486090c5cec07b4894c4"; - Path refFile = new Path(familyDir, hfile); - - // make sure the reference file exists - fs.create(refFile); - - // create the hfile in the archive - fs.mkdirs(archivedHfileDir); - fs.createNewFile(new Path(archivedHfileDir, hfile)); - - // make sure that the file isn't deletable - assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile))); - } - - class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { - public Collection filesUnderSnapshot(final Path snapshotDir) throws IOException { - Collection files = new HashSet(); - files.addAll(SnapshotReferenceUtil.getHFileNames(TEST_UTIL.getConfiguration(), fs, snapshotDir)); - return files; - } - } - - /** - * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException, - * instead of an IOException - */ - @Test - public void testCorruptedRegionManifest() throws IOException { - SnapshotTestingUtils.SnapshotMock - snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( - SNAPSHOT_NAME_STR, TABLE_NAME_STR); - builder.addRegionV2(); - builder.corruptOneRegionManifest(); - - long period = Long.MAX_VALUE; - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, - "test-snapshot-file-cache-refresh", new SnapshotFiles()); - try { - cache.getSnapshotsInProgress(null); - } catch (CorruptedSnapshotException cse) { - LOG.info("Expected exception " + cse); - } finally { - fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); - } - } - - /** - * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException, - * instead of an IOException - */ - @Test - public void testCorruptedDataManifest() throws IOException { - SnapshotTestingUtils.SnapshotMock - snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( - SNAPSHOT_NAME_STR, TABLE_NAME_STR); - builder.addRegionV2(); - // consolidate to generate a data.manifest file - builder.consolidate(); - builder.corruptDataManifest(); - - long period = Long.MAX_VALUE; - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, - "test-snapshot-file-cache-refresh", new SnapshotFiles()); - try { - cache.getSnapshotsInProgress(null); - } catch (CorruptedSnapshotException cse) { - LOG.info("Expected exception " + cse); - } finally { - fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); - } - } - - /** - * HBASE-16464 - */ - @Test - public void testMissedTmpSnapshot() throws IOException { - SnapshotTestingUtils.SnapshotMock - snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( - SNAPSHOT_NAME_STR, TABLE_NAME_STR); - builder.addRegionV2(); - builder.missOneRegionSnapshotFile(); - - long period = Long.MAX_VALUE; - SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, - "test-snapshot-file-cache-refresh", new SnapshotFiles()); - cache.getSnapshotsInProgress(null); - assertFalse(fs.exists(builder.getSnapshotsDir())); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java index 5d056a6..e085347 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java @@ -28,14 +28,15 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotHFileCleaner; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.fs.MasterStorage; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.MetricsMaster; -import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; -import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileCleaner; +import org.apache.hadoop.hbase.fs.legacy.cleaner.HFileLinkCleaner; import org.apache.hadoop.hbase.procedure.ProcedureCoordinator; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.zookeeper.KeeperException; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationHFileCleaner.java new file mode 100644 index 0000000..36d5448 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationHFileCleaner.java @@ -0,0 +1,341 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ +package org.apache.hadoop.hbase.replication.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; + +import com.google.common.collect.Lists; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.replication.ReplicationException; +import org.apache.hadoop.hbase.replication.ReplicationFactory; +import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.ReplicationPeers; +import org.apache.hadoop.hbase.replication.ReplicationQueues; +import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments; +import org.apache.hadoop.hbase.replication.ReplicationQueuesClient; +import org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl; +import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner; +import org.apache.hadoop.hbase.replication.regionserver.Replication; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category({ MasterTests.class, SmallTests.class }) +public class TestReplicationHFileCleaner { + private static final Log LOG = LogFactory.getLog(ReplicationQueuesZKImpl.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static Server server; + private static ReplicationQueues rq; + private static ReplicationPeers rp; + private static final String peerId = "TestReplicationHFileCleaner"; + private static Configuration conf = TEST_UTIL.getConfiguration(); + static FileSystem fs = null; + Path root; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniZKCluster(); + server = new DummyServer(); + conf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); + Replication.decorateMasterConfiguration(conf); + rp = ReplicationFactory.getReplicationPeers(server.getZooKeeper(), conf, server); + rp.init(); + rq = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, server, server.getZooKeeper())); + rq.init(server.getServerName().toString()); + try { + fs = FileSystem.get(conf); + } finally { + if (fs != null) { + fs.close(); + } + } + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniZKCluster(); + } + + @Before + public void setup() throws ReplicationException, IOException { + root = TEST_UTIL.getDataTestDirOnTestFS(); + rp.registerPeer(peerId, new ReplicationPeerConfig().setClusterKey(TEST_UTIL.getClusterKey())); + rq.addPeerToHFileRefs(peerId); + } + + @After + public void cleanup() throws ReplicationException { + try { + fs.delete(root, true); + } catch (IOException e) { + LOG.warn("Failed to delete files recursively from path " + root); + } + rp.unregisterPeer(peerId); + } + + @Test + public void testIsFileDeletable() throws IOException, ReplicationException { + // 1. Create a file + Path file = new Path(root, "testIsFileDeletableWithNoHFileRefs"); + fs.createNewFile(file); + // 2. Assert file is successfully created + assertTrue("Test file not created!", fs.exists(file)); + ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner(); + cleaner.setConf(conf); + // 3. Assert that file as is should be deletable + assertTrue("Cleaner should allow to delete this file as there is no hfile reference node " + + "for it in the queue.", + cleaner.isFileDeletable(fs.getFileStatus(file))); + + List files = new ArrayList(1); + files.add(file.getName()); + // 4. Add the file to hfile-refs queue + rq.addHFileRefs(peerId, files); + // 5. Assert file should not be deletable + assertFalse("Cleaner should not allow to delete this file as there is a hfile reference node " + + "for it in the queue.", + cleaner.isFileDeletable(fs.getFileStatus(file))); + } + + @Test + public void testGetDeletableFiles() throws Exception { + // 1. Create two files and assert that they do not exist + Path notDeletablefile = new Path(root, "testGetDeletableFiles_1"); + fs.createNewFile(notDeletablefile); + assertTrue("Test file not created!", fs.exists(notDeletablefile)); + Path deletablefile = new Path(root, "testGetDeletableFiles_2"); + fs.createNewFile(deletablefile); + assertTrue("Test file not created!", fs.exists(deletablefile)); + + List files = new ArrayList(2); + FileStatus f = new FileStatus(); + f.setPath(deletablefile); + files.add(f); + f = new FileStatus(); + f.setPath(notDeletablefile); + files.add(f); + + List hfiles = new ArrayList<>(1); + hfiles.add(notDeletablefile.getName()); + // 2. Add one file to hfile-refs queue + rq.addHFileRefs(peerId, hfiles); + + ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner(); + cleaner.setConf(conf); + Iterator deletableFilesIterator = cleaner.getDeletableFiles(files).iterator(); + int i = 0; + while (deletableFilesIterator.hasNext() && i < 2) { + i++; + } + // 5. Assert one file should not be deletable and it is present in the list returned + if (i > 2) { + fail("File " + notDeletablefile + + " should not be deletable as its hfile reference node is not added."); + } + assertTrue(deletableFilesIterator.next().getPath().equals(deletablefile)); + } + + /* + * Test for HBASE-14621. This test will not assert directly anything. Without the fix the test + * will end up in a infinite loop, so it will timeout. + */ + @Test(timeout = 15000) + public void testForDifferntHFileRefsZnodeVersion() throws Exception { + // 1. Create a file + Path file = new Path(root, "testForDifferntHFileRefsZnodeVersion"); + fs.createNewFile(file); + // 2. Assert file is successfully created + assertTrue("Test file not created!", fs.exists(file)); + ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner(); + cleaner.setConf(conf); + + ReplicationQueuesClient replicationQueuesClient = Mockito.mock(ReplicationQueuesClient.class); + //Return different znode version for each call + Mockito.when(replicationQueuesClient.getHFileRefsNodeChangeVersion()).thenReturn(1, 2); + + Class cleanerClass = cleaner.getClass(); + Field rqc = cleanerClass.getDeclaredField("rqc"); + rqc.setAccessible(true); + rqc.set(cleaner, replicationQueuesClient); + + cleaner.isFileDeletable(fs.getFileStatus(file)); + } + + /** + * ReplicationHFileCleaner should be able to ride over ZooKeeper errors without aborting. + */ + @Test + public void testZooKeeperAbort() throws Exception { + ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner(); + + List dummyFiles = + Lists.newArrayList(new FileStatus(100, false, 3, 100, System.currentTimeMillis(), new Path( + "hfile1")), new FileStatus(100, false, 3, 100, System.currentTimeMillis(), new Path( + "hfile2"))); + + FaultyZooKeeperWatcher faultyZK = + new FaultyZooKeeperWatcher(conf, "testZooKeeperAbort-faulty", null); + try { + faultyZK.init(); + cleaner.setConf(conf, faultyZK); + // should keep all files due to a ConnectionLossException getting the queues znodes + Iterable toDelete = cleaner.getDeletableFiles(dummyFiles); + assertFalse(toDelete.iterator().hasNext()); + assertFalse(cleaner.isStopped()); + } finally { + faultyZK.close(); + } + + // when zk is working both files should be returned + cleaner = new ReplicationHFileCleaner(); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testZooKeeperAbort-normal", null); + try { + cleaner.setConf(conf, zkw); + Iterable filesToDelete = cleaner.getDeletableFiles(dummyFiles); + Iterator iter = filesToDelete.iterator(); + assertTrue(iter.hasNext()); + assertEquals(new Path("hfile1"), iter.next().getPath()); + assertTrue(iter.hasNext()); + assertEquals(new Path("hfile2"), iter.next().getPath()); + assertFalse(iter.hasNext()); + } finally { + zkw.close(); + } + } + + static class DummyServer implements Server { + + @Override + public Configuration getConfiguration() { + return TEST_UTIL.getConfiguration(); + } + + @Override + public ZooKeeperWatcher getZooKeeper() { + try { + return new ZooKeeperWatcher(getConfiguration(), "dummy server", this); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public CoordinatedStateManager getCoordinatedStateManager() { + return null; + } + + @Override + public ClusterConnection getConnection() { + return null; + } + + @Override + public MetaTableLocator getMetaTableLocator() { + return null; + } + + @Override + public ServerName getServerName() { + return ServerName.valueOf("regionserver,60020,000000"); + } + + @Override + public void abort(String why, Throwable e) { + } + + @Override + public boolean isAborted() { + return false; + } + + @Override + public void stop(String why) { + } + + @Override + public boolean isStopped() { + return false; + } + + @Override + public ChoreService getChoreService() { + return null; + } + + @Override + public ClusterConnection getClusterConnection() { + // TODO Auto-generated method stub + return null; + } + } + + static class FaultyZooKeeperWatcher extends ZooKeeperWatcher { + private RecoverableZooKeeper zk; + public FaultyZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable) + throws ZooKeeperConnectionException, IOException { + super(conf, identifier, abortable); + } + + public void init() throws Exception { + this.zk = spy(super.getRecoverableZooKeeper()); + doThrow(new KeeperException.ConnectionLossException()) + .when(zk).getData("/hbase/replication/hfile-refs", null, new Stat()); + } + + public RecoverableZooKeeper getRecoverableZooKeeper() { + return zk; + } + } +} \ No newline at end of file diff --git a/src/main/asciidoc/_chapters/hbase-default.adoc b/src/main/asciidoc/_chapters/hbase-default.adoc index 60c0849..d94e52d 100644 --- a/src/main/asciidoc/_chapters/hbase-default.adoc +++ b/src/main/asciidoc/_chapters/hbase-default.adoc @@ -143,7 +143,7 @@ A comma-separated list of BaseLogCleanerDelegate invoked by default log cleaners in the list. + .Default -`org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner` +`org.apache.hadoop.hbase.fs.legacy.cleaner.TimeToLiveLogCleaner` [[hbase.master.logcleaner.ttl]] @@ -170,7 +170,7 @@ A comma-separated list of BaseHFileCleanerDelegate invoked by hbase-site.xml. + .Default -`org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner` +`org.apache.hadoop.hbase.fs.legacy.cleaner.TimeToLiveHFileCleaner` [[hbase.master.infoserver.redirect]] -- 2.7.4 (Apple Git-66)