From bee1967eb1073d6a269a944f84319651501aa267 Mon Sep 17 00:00:00 2001 From: Umesh Agashe Date: Tue, 22 Nov 2016 13:31:53 -0800 Subject: [PATCH] HBASE-17164 Moved FileLink and HFileLink to legacy.io package --- .../hbase/fs/legacy/cleaner/HFileCleaner.java | 2 +- .../hbase/fs/legacy/cleaner/HFileLinkCleaner.java | 2 +- .../apache/hadoop/hbase/fs/legacy/io/FileLink.java | 497 +++++++++++++++++++ .../hadoop/hbase/fs/legacy/io/HFileLink.java | 528 +++++++++++++++++++++ .../hadoop/hbase/io/FSDataInputStreamWrapper.java | 1 + .../java/org/apache/hadoop/hbase/io/FileLink.java | 497 ------------------- .../java/org/apache/hadoop/hbase/io/HFileLink.java | 528 --------------------- .../java/org/apache/hadoop/hbase/io/WALLink.java | 1 + .../hbase/mapreduce/LoadIncrementalHFiles.java | 2 +- .../java/org/apache/hadoop/hbase/mob/MobUtils.java | 2 +- .../mob/compactions/PartitionedMobCompactor.java | 2 +- .../hadoop/hbase/mob/mapreduce/SweepJob.java | 2 +- .../hadoop/hbase/mob/mapreduce/SweepReducer.java | 3 +- .../hadoop/hbase/regionserver/StoreFileInfo.java | 2 +- .../hadoop/hbase/snapshot/ExportSnapshot.java | 4 +- .../hbase/snapshot/RestoreSnapshotHelper.java | 2 +- .../apache/hadoop/hbase/snapshot/SnapshotInfo.java | 2 +- .../hbase/snapshot/SnapshotReferenceUtil.java | 2 +- .../hadoop/hbase/util/ServerRegionReplicaUtil.java | 2 +- .../fs/legacy/cleaner/TestHFileLinkCleaner.java | 2 +- .../hadoop/hbase/fs/legacy/io/TestFileLink.java | 287 +++++++++++ .../hadoop/hbase/fs/legacy/io/TestHFileLink.java | 137 ++++++ .../org/apache/hadoop/hbase/io/TestFileLink.java | 286 ----------- .../org/apache/hadoop/hbase/io/TestHFileLink.java | 135 ------ .../TableSnapshotInputFormatTestBase.java | 2 +- .../apache/hadoop/hbase/mob/TestMobFileLink.java | 2 +- .../hbase/mob/compactions/TestMobCompactor.java | 2 +- .../hadoop/hbase/regionserver/TestStoreFile.java | 3 - .../hbase/regionserver/TestStoreFileInfo.java | 2 +- .../hbase/snapshot/SnapshotTestingUtils.java | 4 +- .../hbase/snapshot/TestRestoreSnapshotHelper.java | 2 +- 31 files changed, 1472 insertions(+), 1473 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/io/FileLink.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/io/HFileLink.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/io/TestFileLink.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/io/TestHFileLink.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileCleaner.java index 7137432..b58f80e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileCleaner.java @@ -25,7 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Stoppable; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; /** * This Chore, every time it runs, will clear the HFiles in the hfile archive diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileLinkCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileLinkCleaner.java index 7c1f4d2..76a109a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileLinkCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/cleaner/HFileLinkCleaner.java @@ -27,7 +27,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.util.FSUtils; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/io/FileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/io/FileLink.java new file mode 100644 index 0000000..5d46bc5 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/io/FileLink.java @@ -0,0 +1,497 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.io; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.io.IOException; +import java.io.InputStream; +import java.io.FileNotFoundException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.fs.Seekable; +import org.apache.hadoop.hbase.util.FSUtils; + +/** + * The FileLink is a sort of hardlink, that allows access to a file given a set of locations. + * + *

The Problem: + *

+ * If we want to create a reference to a file, we need to remember that it can be in its + * original location or in the archive folder. + * The FileLink class tries to abstract this concept and given a set of locations + * it is able to switch between them making this operation transparent for the user. + * {@link HFileLink} is a more concrete implementation of the {@code FileLink}. + * + *

Back-references: + * To help the {@link org.apache.hadoop.hbase.fs.legacy.cleaner.CleanerChore} to keep track of + * the links to a particular file, during the {@code FileLink} creation, a new file is placed + * inside a back-reference directory. There's one back-reference directory for each file that + * has links, and in the directory there's one file per link. + * + *

HFileLink Example + *

+ */ +@InterfaceAudience.Private +public class FileLink { + private static final Log LOG = LogFactory.getLog(FileLink.class); + + /** Define the Back-reference directory name prefix: .links-<hfile>/ */ + public static final String BACK_REFERENCES_DIRECTORY_PREFIX = ".links-"; + + /** + * FileLink InputStream that handles the switch between the original path + * and the alternative locations, when the file is moved. + */ + private static class FileLinkInputStream extends InputStream + implements Seekable, PositionedReadable { + private FSDataInputStream in = null; + private Path currentPath = null; + private long pos = 0; + + private final FileLink fileLink; + private final int bufferSize; + private final FileSystem fs; + + public FileLinkInputStream(final FileSystem fs, final FileLink fileLink) + throws IOException { + this(fs, fileLink, FSUtils.getDefaultBufferSize(fs)); + } + + public FileLinkInputStream(final FileSystem fs, final FileLink fileLink, int bufferSize) + throws IOException { + this.bufferSize = bufferSize; + this.fileLink = fileLink; + this.fs = fs; + + this.in = tryOpen(); + } + + @Override + public int read() throws IOException { + int res; + try { + res = in.read(); + } catch (FileNotFoundException e) { + res = tryOpen().read(); + } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() + res = tryOpen().read(); + } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() + res = tryOpen().read(); + } + if (res > 0) pos += 1; + return res; + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int n; + try { + n = in.read(b, off, len); + } catch (FileNotFoundException e) { + n = tryOpen().read(b, off, len); + } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() + n = tryOpen().read(b, off, len); + } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() + n = tryOpen().read(b, off, len); + } + if (n > 0) pos += n; + assert(in.getPos() == pos); + return n; + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) throws IOException { + int n; + try { + n = in.read(position, buffer, offset, length); + } catch (FileNotFoundException e) { + n = tryOpen().read(position, buffer, offset, length); + } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() + n = tryOpen().read(position, buffer, offset, length); + } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() + n = tryOpen().read(position, buffer, offset, length); + } + return n; + } + + @Override + public void readFully(long position, byte[] buffer) throws IOException { + readFully(position, buffer, 0, buffer.length); + } + + @Override + public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { + try { + in.readFully(position, buffer, offset, length); + } catch (FileNotFoundException e) { + tryOpen().readFully(position, buffer, offset, length); + } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() + tryOpen().readFully(position, buffer, offset, length); + } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() + tryOpen().readFully(position, buffer, offset, length); + } + } + + @Override + public long skip(long n) throws IOException { + long skipped; + + try { + skipped = in.skip(n); + } catch (FileNotFoundException e) { + skipped = tryOpen().skip(n); + } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() + skipped = tryOpen().skip(n); + } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() + skipped = tryOpen().skip(n); + } + + if (skipped > 0) pos += skipped; + return skipped; + } + + @Override + public int available() throws IOException { + try { + return in.available(); + } catch (FileNotFoundException e) { + return tryOpen().available(); + } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() + return tryOpen().available(); + } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() + return tryOpen().available(); + } + } + + @Override + public void seek(long pos) throws IOException { + try { + in.seek(pos); + } catch (FileNotFoundException e) { + tryOpen().seek(pos); + } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() + tryOpen().seek(pos); + } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() + tryOpen().seek(pos); + } + this.pos = pos; + } + + @Override + public long getPos() throws IOException { + return pos; + } + + @Override + public boolean seekToNewSource(long targetPos) throws IOException { + boolean res; + try { + res = in.seekToNewSource(targetPos); + } catch (FileNotFoundException e) { + res = tryOpen().seekToNewSource(targetPos); + } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() + res = tryOpen().seekToNewSource(targetPos); + } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() + res = tryOpen().seekToNewSource(targetPos); + } + if (res) pos = targetPos; + return res; + } + + @Override + public void close() throws IOException { + in.close(); + } + + @Override + public synchronized void mark(int readlimit) { + } + + @Override + public synchronized void reset() throws IOException { + throw new IOException("mark/reset not supported"); + } + + @Override + public boolean markSupported() { + return false; + } + + /** + * Try to open the file from one of the available locations. + * + * @return FSDataInputStream stream of the opened file link + * @throws IOException on unexpected error, or file not found. + */ + private FSDataInputStream tryOpen() throws IOException { + for (Path path: fileLink.getLocations()) { + if (path.equals(currentPath)) continue; + try { + in = fs.open(path, bufferSize); + if (pos != 0) in.seek(pos); + assert(in.getPos() == pos) : "Link unable to seek to the right position=" + pos; + if (LOG.isTraceEnabled()) { + if (currentPath == null) { + LOG.debug("link open path=" + path); + } else { + LOG.trace("link switch from path=" + currentPath + " to path=" + path); + } + } + currentPath = path; + return(in); + } catch (FileNotFoundException e) { + // Try another file location + } + } + throw new FileNotFoundException("Unable to open link: " + fileLink); + } + } + + private Path[] locations = null; + + protected FileLink() { + this.locations = null; + } + + /** + * @param originPath Original location of the file to link + * @param alternativePaths Alternative locations to look for the linked file + */ + public FileLink(Path originPath, Path... alternativePaths) { + setLocations(originPath, alternativePaths); + } + + /** + * @param locations locations to look for the linked file + */ + public FileLink(final Collection locations) { + this.locations = locations.toArray(new Path[locations.size()]); + } + + /** + * @return the locations to look for the linked file. + */ + public Path[] getLocations() { + return locations; + } + + @Override + public String toString() { + StringBuilder str = new StringBuilder(getClass().getName()); + str.append(" locations=["); + for (int i = 0; i < locations.length; ++i) { + if (i > 0) str.append(", "); + str.append(locations[i].toString()); + } + str.append("]"); + return str.toString(); + } + + /** + * @return true if the file pointed by the link exists + */ + public boolean exists(final FileSystem fs) throws IOException { + for (int i = 0; i < locations.length; ++i) { + if (fs.exists(locations[i])) { + return true; + } + } + return false; + } + + /** + * @return the path of the first available link. + */ + public Path getAvailablePath(FileSystem fs) throws IOException { + for (int i = 0; i < locations.length; ++i) { + if (fs.exists(locations[i])) { + return locations[i]; + } + } + throw new FileNotFoundException("Unable to open link: " + this); + } + + /** + * Get the FileStatus of the referenced file. + * + * @param fs {@link FileSystem} on which to get the file status + * @return InputStream for the hfile link. + * @throws IOException on unexpected error. + */ + public FileStatus getFileStatus(FileSystem fs) throws IOException { + for (int i = 0; i < locations.length; ++i) { + try { + return fs.getFileStatus(locations[i]); + } catch (FileNotFoundException e) { + // Try another file location + } + } + throw new FileNotFoundException("Unable to open link: " + this); + } + + /** + * Open the FileLink for read. + *

+ * It uses a wrapper of FSDataInputStream that is agnostic to the location + * of the file, even if the file switches between locations. + * + * @param fs {@link FileSystem} on which to open the FileLink + * @return InputStream for reading the file link. + * @throws IOException on unexpected error. + */ + public FSDataInputStream open(final FileSystem fs) throws IOException { + return new FSDataInputStream(new FileLinkInputStream(fs, this)); + } + + /** + * Open the FileLink for read. + *

+ * It uses a wrapper of FSDataInputStream that is agnostic to the location + * of the file, even if the file switches between locations. + * + * @param fs {@link FileSystem} on which to open the FileLink + * @param bufferSize the size of the buffer to be used. + * @return InputStream for reading the file link. + * @throws IOException on unexpected error. + */ + public FSDataInputStream open(final FileSystem fs, int bufferSize) throws IOException { + return new FSDataInputStream(new FileLinkInputStream(fs, this, bufferSize)); + } + + /** + * NOTE: This method must be used only in the constructor! + * It creates a List with the specified locations for the link. + */ + protected void setLocations(Path originPath, Path... alternativePaths) { + assert this.locations == null : "Link locations already set"; + + List paths = new ArrayList(alternativePaths.length +1); + if (originPath != null) { + paths.add(originPath); + } + + for (int i = 0; i < alternativePaths.length; i++) { + if (alternativePaths[i] != null) { + paths.add(alternativePaths[i]); + } + } + this.locations = paths.toArray(new Path[0]); + } + + /** + * Get the directory to store the link back references + * + *

To simplify the reference count process, during the FileLink creation + * a back-reference is added to the back-reference directory of the specified file. + * + * @param storeDir Root directory for the link reference folder + * @param fileName File Name with links + * @return Path for the link back references. + */ + public static Path getBackReferencesDir(final Path storeDir, final String fileName) { + return new Path(storeDir, BACK_REFERENCES_DIRECTORY_PREFIX + fileName); + } + + /** + * Get the referenced file name from the reference link directory path. + * + * @param dirPath Link references directory path + * @return Name of the file referenced + */ + public static String getBackReferenceFileName(final Path dirPath) { + return dirPath.getName().substring(BACK_REFERENCES_DIRECTORY_PREFIX.length()); + } + + /** + * Checks if the specified directory path is a back reference links folder. + * + * @param dirPath Directory path to verify + * @return True if the specified directory is a link references folder + */ + public static boolean isBackReferencesDir(final Path dirPath) { + if (dirPath == null) return false; + return dirPath.getName().startsWith(BACK_REFERENCES_DIRECTORY_PREFIX); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + // Assumes that the ordering of locations between objects are the same. This is true for the + // current subclasses already (HFileLink, WALLink). Otherwise, we may have to sort the locations + // or keep them presorted + if (this.getClass().equals(obj.getClass())) { + return Arrays.equals(this.locations, ((FileLink) obj).locations); + } + + return false; + } + + @Override + public int hashCode() { + return Arrays.hashCode(locations); + } +} + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/io/HFileLink.java new file mode 100644 index 0000000..18f5d3d --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/legacy/io/HFileLink.java @@ -0,0 +1,528 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.io; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.mob.MobConstants; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.apache.hadoop.hbase.util.Pair; + +/** + * HFileLink describes a link to an hfile. + * + * An hfile can be served from a region or from the hfile archive directory (/hbase/.archive) + * HFileLink allows to access the referenced hfile regardless of the location where it is. + * + *

Searches for hfiles in the following order and locations: + *

+ * + * The link checks first in the original path if it is not present + * it fallbacks to the archived path. + */ +@InterfaceAudience.Private +@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="EQ_DOESNT_OVERRIDE_EQUALS", + justification="To be fixed but warning suppressed for now") +public class HFileLink extends FileLink { + private static final Log LOG = LogFactory.getLog(HFileLink.class); + + /** + * A non-capture group, for HFileLink, so that this can be embedded. + * The HFileLink describe a link to an hfile in a different table/region + * and the name is in the form: table=region-hfile. + *

+ * Table name is ([a-zA-Z_0-9][a-zA-Z_0-9.-]*), so '=' is an invalid character for the table name. + * Region name is ([a-f0-9]+), so '-' is an invalid character for the region name. + * HFile is ([0-9a-f]+(?:_SeqId_[0-9]+_)?) covering the plain hfiles (uuid) + * and the bulk loaded (_SeqId_[0-9]+_) hfiles. + */ + public static final String LINK_NAME_REGEX = + String.format("(?:(?:%s=)?)%s=%s-%s", + TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, + HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFileInfo.HFILE_NAME_REGEX); + + /** Define the HFile Link name parser in the form of: table=region-hfile */ + //made package private for testing + static final Pattern LINK_NAME_PATTERN = + Pattern.compile(String.format("^(?:(%s)(?:\\=))?(%s)=(%s)-(%s)$", + TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, + HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFileInfo.HFILE_NAME_REGEX)); + + /** + * The pattern should be used for hfile and reference links + * that can be found in /hbase/table/region/family/ + */ + private static final Pattern REF_OR_HFILE_LINK_PATTERN = + Pattern.compile(String.format("^(?:(%s)(?:=))?(%s)=(%s)-(.+)$", + TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, + HRegionInfo.ENCODED_REGION_NAME_REGEX)); + + private final Path archivePath; + private final Path originPath; + private final Path mobPath; + private final Path tempPath; + + /** + * Dead simple hfile link constructor + */ + public HFileLink(final Path originPath, final Path tempPath, final Path mobPath, + final Path archivePath) { + this.tempPath = tempPath; + this.originPath = originPath; + this.mobPath = mobPath; + this.archivePath = archivePath; + setLocations(originPath, tempPath, mobPath, archivePath); + } + + + /** + * @param conf {@link Configuration} from which to extract specific archive locations + * @param hFileLinkPattern The path ending with a HFileLink pattern. (table=region-hfile) + * @throws IOException on unexpected error. + */ + public static final HFileLink buildFromHFileLinkPattern(Configuration conf, Path hFileLinkPattern) + throws IOException { + return buildFromHFileLinkPattern(FSUtils.getRootDir(conf), + HFileArchiveUtil.getArchivePath(conf), hFileLinkPattern); + } + + + + /** + * @param rootDir Path to the root directory where hbase files are stored + * @param archiveDir Path to the hbase archive directory + * @param hFileLinkPattern The path of the HFile Link. + */ + public final static HFileLink buildFromHFileLinkPattern(final Path rootDir, + final Path archiveDir, + final Path hFileLinkPattern) { + Path hfilePath = getHFileLinkPatternRelativePath(hFileLinkPattern); + Path tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath); + Path originPath = new Path(rootDir, hfilePath); + Path mobPath = new Path(new Path(rootDir, MobConstants.MOB_DIR_NAME), hfilePath); + Path archivePath = new Path(archiveDir, hfilePath); + return new HFileLink(originPath, tempPath, mobPath, archivePath); + } + + /** + * Create an HFileLink relative path for the table/region/family/hfile location + * @param table Table name + * @param region Region Name + * @param family Family Name + * @param hfile HFile Name + * @return the relative Path to open the specified table/region/family/hfile link + */ + public static Path createPath(final TableName table, final String region, + final String family, final String hfile) { + if (HFileLink.isHFileLink(hfile)) { + return new Path(family, hfile); + } + return new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); + } + + /** + * Create an HFileLink instance from table/region/family/hfile location + * @param conf {@link Configuration} from which to extract specific archive locations + * @param table Table name + * @param region Region Name + * @param family Family Name + * @param hfile HFile Name + * @return Link to the file with the specified table/region/family/hfile location + * @throws IOException on unexpected error. + */ + public static HFileLink build(final Configuration conf, final TableName table, + final String region, final String family, final String hfile) + throws IOException { + return HFileLink.buildFromHFileLinkPattern(conf, createPath(table, region, family, hfile)); + } + + /** + * @return the origin path of the hfile. + */ + public Path getOriginPath() { + return this.originPath; + } + + /** + * @return the path of the archived hfile. + */ + public Path getArchivePath() { + return this.archivePath; + } + + /** + * @return the path of the mob hfiles. + */ + public Path getMobPath() { + return this.mobPath; + } + + /** + * @param path Path to check. + * @return True if the path is a HFileLink. + */ + public static boolean isHFileLink(final Path path) { + return isHFileLink(path.getName()); + } + + + /** + * @param fileName File name to check. + * @return True if the path is a HFileLink. + */ + public static boolean isHFileLink(String fileName) { + Matcher m = LINK_NAME_PATTERN.matcher(fileName); + if (!m.matches()) return false; + return m.groupCount() > 2 && m.group(4) != null && m.group(3) != null && m.group(2) != null; + } + + /** + * Convert a HFileLink path to a table relative path. + * e.g. the link: /hbase/test/0123/cf/testtb=4567-abcd + * becomes: /hbase/testtb/4567/cf/abcd + * + * @param path HFileLink path + * @return Relative table path + * @throws IOException on unexpected error. + */ + private static Path getHFileLinkPatternRelativePath(final Path path) { + // table=region-hfile + Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(path.getName()); + if (!m.matches()) { + throw new IllegalArgumentException(path.getName() + " is not a valid HFileLink pattern!"); + } + + // Convert the HFileLink name into a real table/region/cf/hfile path. + TableName tableName = TableName.valueOf(m.group(1), m.group(2)); + String regionName = m.group(3); + String hfileName = m.group(4); + String familyName = path.getParent().getName(); + Path tableDir = FSUtils.getTableDir(new Path("./"), tableName); + return new Path(tableDir, new Path(regionName, new Path(familyName, + hfileName))); + } + + /** + * Get the HFile name of the referenced link + * + * @param fileName HFileLink file name + * @return the name of the referenced HFile + */ + public static String getReferencedHFileName(final String fileName) { + Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName); + if (!m.matches()) { + throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!"); + } + return(m.group(4)); + } + + /** + * Get the Region name of the referenced link + * + * @param fileName HFileLink file name + * @return the name of the referenced Region + */ + public static String getReferencedRegionName(final String fileName) { + Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName); + if (!m.matches()) { + throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!"); + } + return(m.group(3)); + } + + /** + * Get the Table name of the referenced link + * + * @param fileName HFileLink file name + * @return the name of the referenced Table + */ + public static TableName getReferencedTableName(final String fileName) { + Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName); + if (!m.matches()) { + throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!"); + } + return(TableName.valueOf(m.group(1), m.group(2))); + } + + /** + * Create a new HFileLink name + * + * @param hfileRegionInfo - Linked HFile Region Info + * @param hfileName - Linked HFile name + * @return file name of the HFile Link + */ + public static String createHFileLinkName(final HRegionInfo hfileRegionInfo, + final String hfileName) { + return createHFileLinkName(hfileRegionInfo.getTable(), + hfileRegionInfo.getEncodedName(), hfileName); + } + + /** + * Create a new HFileLink name + * + * @param tableName - Linked HFile table name + * @param regionName - Linked HFile region name + * @param hfileName - Linked HFile name + * @return file name of the HFile Link + */ + public static String createHFileLinkName(final TableName tableName, + final String regionName, final String hfileName) { + String s = String.format("%s=%s-%s", + tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '='), + regionName, hfileName); + return s; + } + + /** + * Create a new HFileLink + * + *

It also adds a back-reference to the hfile back-reference directory + * to simplify the reference-count and the cleaning process. + * + * @param conf {@link Configuration} to read for the archive directory name + * @param fs {@link FileSystem} on which to write the HFileLink + * @param dstFamilyPath - Destination path (table/region/cf/) + * @param hfileRegionInfo - Linked HFile Region Info + * @param hfileName - Linked HFile name + * @return true if the file is created, otherwise the file exists. + * @throws IOException on file or parent directory creation failure + */ + public static boolean create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, + final String hfileName) throws IOException { + return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true); + } + + /** + * Create a new HFileLink + * + *

It also adds a back-reference to the hfile back-reference directory + * to simplify the reference-count and the cleaning process. + * + * @param conf {@link Configuration} to read for the archive directory name + * @param fs {@link FileSystem} on which to write the HFileLink + * @param dstFamilyPath - Destination path (table/region/cf/) + * @param hfileRegionInfo - Linked HFile Region Info + * @param hfileName - Linked HFile name + * @param createBackRef - Whether back reference should be created. Defaults to true. + * @return true if the file is created, otherwise the file exists. + * @throws IOException on file or parent directory creation failure + */ + public static boolean create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, + final String hfileName, final boolean createBackRef) throws IOException { + TableName linkedTable = hfileRegionInfo.getTable(); + String linkedRegion = hfileRegionInfo.getEncodedName(); + return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef); + } + + /** + * Create a new HFileLink + * + *

It also adds a back-reference to the hfile back-reference directory + * to simplify the reference-count and the cleaning process. + * + * @param conf {@link Configuration} to read for the archive directory name + * @param fs {@link FileSystem} on which to write the HFileLink + * @param dstFamilyPath - Destination path (table/region/cf/) + * @param linkedTable - Linked Table Name + * @param linkedRegion - Linked Region Name + * @param hfileName - Linked HFile name + * @return true if the file is created, otherwise the file exists. + * @throws IOException on file or parent directory creation failure + */ + public static boolean create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, + final String hfileName) throws IOException { + return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true); + } + + /** + * Create a new HFileLink + * + *

It also adds a back-reference to the hfile back-reference directory + * to simplify the reference-count and the cleaning process. + * + * @param conf {@link Configuration} to read for the archive directory name + * @param fs {@link FileSystem} on which to write the HFileLink + * @param dstFamilyPath - Destination path (table/region/cf/) + * @param linkedTable - Linked Table Name + * @param linkedRegion - Linked Region Name + * @param hfileName - Linked HFile name + * @param createBackRef - Whether back reference should be created. Defaults to true. + * @return true if the file is created, otherwise the file exists. + * @throws IOException on file or parent directory creation failure + */ + public static boolean create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, + final String hfileName, final boolean createBackRef) throws IOException { + String familyName = dstFamilyPath.getName(); + String regionName = dstFamilyPath.getParent().getName(); + String tableName = FSUtils.getTableName(dstFamilyPath.getParent().getParent()) + .getNameAsString(); + + String name = createHFileLinkName(linkedTable, linkedRegion, hfileName); + String refName = createBackReferenceName(tableName, regionName); + + // Make sure the destination directory exists + fs.mkdirs(dstFamilyPath); + + // Make sure the FileLink reference directory exists + Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, + linkedTable, linkedRegion, familyName); + Path backRefPath = null; + if (createBackRef) { + Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName); + fs.mkdirs(backRefssDir); + + // Create the reference for the link + backRefPath = new Path(backRefssDir, refName); + fs.createNewFile(backRefPath); + } + try { + // Create the link + return fs.createNewFile(new Path(dstFamilyPath, name)); + } catch (IOException e) { + LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e); + // Revert the reference if the link creation failed + if (createBackRef) { + fs.delete(backRefPath, false); + } + throw e; + } + } + + /** + * Create a new HFileLink starting from a hfileLink name + * + *

It also adds a back-reference to the hfile back-reference directory + * to simplify the reference-count and the cleaning process. + * + * @param conf {@link Configuration} to read for the archive directory name + * @param fs {@link FileSystem} on which to write the HFileLink + * @param dstFamilyPath - Destination path (table/region/cf/) + * @param hfileLinkName - HFileLink name (it contains hfile-region-table) + * @return true if the file is created, otherwise the file exists. + * @throws IOException on file or parent directory creation failure + */ + public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final String hfileLinkName) + throws IOException { + return createFromHFileLink(conf, fs, dstFamilyPath, hfileLinkName, true); + } + + /** + * Create a new HFileLink starting from a hfileLink name + * + *

It also adds a back-reference to the hfile back-reference directory + * to simplify the reference-count and the cleaning process. + * + * @param conf {@link Configuration} to read for the archive directory name + * @param fs {@link FileSystem} on which to write the HFileLink + * @param dstFamilyPath - Destination path (table/region/cf/) + * @param hfileLinkName - HFileLink name (it contains hfile-region-table) + * @param createBackRef - Whether back reference should be created. Defaults to true. + * @return true if the file is created, otherwise the file exists. + * @throws IOException on file or parent directory creation failure + */ + public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef) + throws IOException { + Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName); + if (!m.matches()) { + throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!"); + } + return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)), + m.group(3), m.group(4), createBackRef); + } + + /** + * Create the back reference name + */ + //package-private for testing + static String createBackReferenceName(final String tableNameStr, + final String regionName) { + + return regionName + "." + tableNameStr.replace(TableName.NAMESPACE_DELIM, '='); + } + + /** + * Get the full path of the HFile referenced by the back reference + * + * @param rootDir root hbase directory + * @param linkRefPath Link Back Reference path + * @return full path of the referenced hfile + */ + public static Path getHFileFromBackReference(final Path rootDir, final Path linkRefPath) { + Pair p = parseBackReferenceName(linkRefPath.getName()); + TableName linkTableName = p.getFirst(); + String linkRegionName = p.getSecond(); + + String hfileName = getBackReferenceFileName(linkRefPath.getParent()); + Path familyPath = linkRefPath.getParent().getParent(); + Path regionPath = familyPath.getParent(); + Path tablePath = regionPath.getParent(); + + String linkName = createHFileLinkName(FSUtils.getTableName(tablePath), + regionPath.getName(), hfileName); + Path linkTableDir = FSUtils.getTableDir(rootDir, linkTableName); + Path regionDir = HRegion.getRegionDir(linkTableDir, linkRegionName); + return new Path(new Path(regionDir, familyPath.getName()), linkName); + } + + static Pair parseBackReferenceName(String name) { + int separatorIndex = name.indexOf('.'); + String linkRegionName = name.substring(0, separatorIndex); + String tableSubstr = name.substring(separatorIndex + 1) + .replace('=', TableName.NAMESPACE_DELIM); + TableName linkTableName = TableName.valueOf(tableSubstr); + return new Pair(linkTableName, linkRegionName); + } + + /** + * Get the full path of the HFile referenced by the back reference + * + * @param conf {@link Configuration} to read for the archive directory name + * @param linkRefPath Link Back Reference path + * @return full path of the referenced hfile + * @throws IOException on unexpected error. + */ + public static Path getHFileFromBackReference(final Configuration conf, final Path linkRefPath) + throws IOException { + return getHFileFromBackReference(FSUtils.getRootDir(conf), linkRefPath); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java index b06be6b..ac0e69e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.fs.HFileSystem; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hbase.fs.legacy.io.FileLink; /** * Wrapper for input stream(s) that takes care of the interaction of FS and HBase checksums, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java deleted file mode 100644 index d3acb05..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java +++ /dev/null @@ -1,497 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.io; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.io.IOException; -import java.io.InputStream; -import java.io.FileNotFoundException; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PositionedReadable; -import org.apache.hadoop.fs.Seekable; -import org.apache.hadoop.hbase.util.FSUtils; - -/** - * The FileLink is a sort of hardlink, that allows access to a file given a set of locations. - * - *

The Problem: - *

- * If we want to create a reference to a file, we need to remember that it can be in its - * original location or in the archive folder. - * The FileLink class tries to abstract this concept and given a set of locations - * it is able to switch between them making this operation transparent for the user. - * {@link HFileLink} is a more concrete implementation of the {@code FileLink}. - * - *

Back-references: - * To help the {@link org.apache.hadoop.hbase.fs.legacy.cleaner.CleanerChore} to keep track of - * the links to a particular file, during the {@code FileLink} creation, a new file is placed - * inside a back-reference directory. There's one back-reference directory for each file that - * has links, and in the directory there's one file per link. - * - *

HFileLink Example - *

- */ -@InterfaceAudience.Private -public class FileLink { - private static final Log LOG = LogFactory.getLog(FileLink.class); - - /** Define the Back-reference directory name prefix: .links-<hfile>/ */ - public static final String BACK_REFERENCES_DIRECTORY_PREFIX = ".links-"; - - /** - * FileLink InputStream that handles the switch between the original path - * and the alternative locations, when the file is moved. - */ - private static class FileLinkInputStream extends InputStream - implements Seekable, PositionedReadable { - private FSDataInputStream in = null; - private Path currentPath = null; - private long pos = 0; - - private final FileLink fileLink; - private final int bufferSize; - private final FileSystem fs; - - public FileLinkInputStream(final FileSystem fs, final FileLink fileLink) - throws IOException { - this(fs, fileLink, FSUtils.getDefaultBufferSize(fs)); - } - - public FileLinkInputStream(final FileSystem fs, final FileLink fileLink, int bufferSize) - throws IOException { - this.bufferSize = bufferSize; - this.fileLink = fileLink; - this.fs = fs; - - this.in = tryOpen(); - } - - @Override - public int read() throws IOException { - int res; - try { - res = in.read(); - } catch (FileNotFoundException e) { - res = tryOpen().read(); - } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() - res = tryOpen().read(); - } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() - res = tryOpen().read(); - } - if (res > 0) pos += 1; - return res; - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int n; - try { - n = in.read(b, off, len); - } catch (FileNotFoundException e) { - n = tryOpen().read(b, off, len); - } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() - n = tryOpen().read(b, off, len); - } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() - n = tryOpen().read(b, off, len); - } - if (n > 0) pos += n; - assert(in.getPos() == pos); - return n; - } - - @Override - public int read(long position, byte[] buffer, int offset, int length) throws IOException { - int n; - try { - n = in.read(position, buffer, offset, length); - } catch (FileNotFoundException e) { - n = tryOpen().read(position, buffer, offset, length); - } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() - n = tryOpen().read(position, buffer, offset, length); - } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() - n = tryOpen().read(position, buffer, offset, length); - } - return n; - } - - @Override - public void readFully(long position, byte[] buffer) throws IOException { - readFully(position, buffer, 0, buffer.length); - } - - @Override - public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { - try { - in.readFully(position, buffer, offset, length); - } catch (FileNotFoundException e) { - tryOpen().readFully(position, buffer, offset, length); - } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() - tryOpen().readFully(position, buffer, offset, length); - } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() - tryOpen().readFully(position, buffer, offset, length); - } - } - - @Override - public long skip(long n) throws IOException { - long skipped; - - try { - skipped = in.skip(n); - } catch (FileNotFoundException e) { - skipped = tryOpen().skip(n); - } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() - skipped = tryOpen().skip(n); - } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() - skipped = tryOpen().skip(n); - } - - if (skipped > 0) pos += skipped; - return skipped; - } - - @Override - public int available() throws IOException { - try { - return in.available(); - } catch (FileNotFoundException e) { - return tryOpen().available(); - } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() - return tryOpen().available(); - } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() - return tryOpen().available(); - } - } - - @Override - public void seek(long pos) throws IOException { - try { - in.seek(pos); - } catch (FileNotFoundException e) { - tryOpen().seek(pos); - } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() - tryOpen().seek(pos); - } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() - tryOpen().seek(pos); - } - this.pos = pos; - } - - @Override - public long getPos() throws IOException { - return pos; - } - - @Override - public boolean seekToNewSource(long targetPos) throws IOException { - boolean res; - try { - res = in.seekToNewSource(targetPos); - } catch (FileNotFoundException e) { - res = tryOpen().seekToNewSource(targetPos); - } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() - res = tryOpen().seekToNewSource(targetPos); - } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() - res = tryOpen().seekToNewSource(targetPos); - } - if (res) pos = targetPos; - return res; - } - - @Override - public void close() throws IOException { - in.close(); - } - - @Override - public synchronized void mark(int readlimit) { - } - - @Override - public synchronized void reset() throws IOException { - throw new IOException("mark/reset not supported"); - } - - @Override - public boolean markSupported() { - return false; - } - - /** - * Try to open the file from one of the available locations. - * - * @return FSDataInputStream stream of the opened file link - * @throws IOException on unexpected error, or file not found. - */ - private FSDataInputStream tryOpen() throws IOException { - for (Path path: fileLink.getLocations()) { - if (path.equals(currentPath)) continue; - try { - in = fs.open(path, bufferSize); - if (pos != 0) in.seek(pos); - assert(in.getPos() == pos) : "Link unable to seek to the right position=" + pos; - if (LOG.isTraceEnabled()) { - if (currentPath == null) { - LOG.debug("link open path=" + path); - } else { - LOG.trace("link switch from path=" + currentPath + " to path=" + path); - } - } - currentPath = path; - return(in); - } catch (FileNotFoundException e) { - // Try another file location - } - } - throw new FileNotFoundException("Unable to open link: " + fileLink); - } - } - - private Path[] locations = null; - - protected FileLink() { - this.locations = null; - } - - /** - * @param originPath Original location of the file to link - * @param alternativePaths Alternative locations to look for the linked file - */ - public FileLink(Path originPath, Path... alternativePaths) { - setLocations(originPath, alternativePaths); - } - - /** - * @param locations locations to look for the linked file - */ - public FileLink(final Collection locations) { - this.locations = locations.toArray(new Path[locations.size()]); - } - - /** - * @return the locations to look for the linked file. - */ - public Path[] getLocations() { - return locations; - } - - @Override - public String toString() { - StringBuilder str = new StringBuilder(getClass().getName()); - str.append(" locations=["); - for (int i = 0; i < locations.length; ++i) { - if (i > 0) str.append(", "); - str.append(locations[i].toString()); - } - str.append("]"); - return str.toString(); - } - - /** - * @return true if the file pointed by the link exists - */ - public boolean exists(final FileSystem fs) throws IOException { - for (int i = 0; i < locations.length; ++i) { - if (fs.exists(locations[i])) { - return true; - } - } - return false; - } - - /** - * @return the path of the first available link. - */ - public Path getAvailablePath(FileSystem fs) throws IOException { - for (int i = 0; i < locations.length; ++i) { - if (fs.exists(locations[i])) { - return locations[i]; - } - } - throw new FileNotFoundException("Unable to open link: " + this); - } - - /** - * Get the FileStatus of the referenced file. - * - * @param fs {@link FileSystem} on which to get the file status - * @return InputStream for the hfile link. - * @throws IOException on unexpected error. - */ - public FileStatus getFileStatus(FileSystem fs) throws IOException { - for (int i = 0; i < locations.length; ++i) { - try { - return fs.getFileStatus(locations[i]); - } catch (FileNotFoundException e) { - // Try another file location - } - } - throw new FileNotFoundException("Unable to open link: " + this); - } - - /** - * Open the FileLink for read. - *

- * It uses a wrapper of FSDataInputStream that is agnostic to the location - * of the file, even if the file switches between locations. - * - * @param fs {@link FileSystem} on which to open the FileLink - * @return InputStream for reading the file link. - * @throws IOException on unexpected error. - */ - public FSDataInputStream open(final FileSystem fs) throws IOException { - return new FSDataInputStream(new FileLinkInputStream(fs, this)); - } - - /** - * Open the FileLink for read. - *

- * It uses a wrapper of FSDataInputStream that is agnostic to the location - * of the file, even if the file switches between locations. - * - * @param fs {@link FileSystem} on which to open the FileLink - * @param bufferSize the size of the buffer to be used. - * @return InputStream for reading the file link. - * @throws IOException on unexpected error. - */ - public FSDataInputStream open(final FileSystem fs, int bufferSize) throws IOException { - return new FSDataInputStream(new FileLinkInputStream(fs, this, bufferSize)); - } - - /** - * NOTE: This method must be used only in the constructor! - * It creates a List with the specified locations for the link. - */ - protected void setLocations(Path originPath, Path... alternativePaths) { - assert this.locations == null : "Link locations already set"; - - List paths = new ArrayList(alternativePaths.length +1); - if (originPath != null) { - paths.add(originPath); - } - - for (int i = 0; i < alternativePaths.length; i++) { - if (alternativePaths[i] != null) { - paths.add(alternativePaths[i]); - } - } - this.locations = paths.toArray(new Path[0]); - } - - /** - * Get the directory to store the link back references - * - *

To simplify the reference count process, during the FileLink creation - * a back-reference is added to the back-reference directory of the specified file. - * - * @param storeDir Root directory for the link reference folder - * @param fileName File Name with links - * @return Path for the link back references. - */ - public static Path getBackReferencesDir(final Path storeDir, final String fileName) { - return new Path(storeDir, BACK_REFERENCES_DIRECTORY_PREFIX + fileName); - } - - /** - * Get the referenced file name from the reference link directory path. - * - * @param dirPath Link references directory path - * @return Name of the file referenced - */ - public static String getBackReferenceFileName(final Path dirPath) { - return dirPath.getName().substring(BACK_REFERENCES_DIRECTORY_PREFIX.length()); - } - - /** - * Checks if the specified directory path is a back reference links folder. - * - * @param dirPath Directory path to verify - * @return True if the specified directory is a link references folder - */ - public static boolean isBackReferencesDir(final Path dirPath) { - if (dirPath == null) return false; - return dirPath.getName().startsWith(BACK_REFERENCES_DIRECTORY_PREFIX); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - // Assumes that the ordering of locations between objects are the same. This is true for the - // current subclasses already (HFileLink, WALLink). Otherwise, we may have to sort the locations - // or keep them presorted - if (this.getClass().equals(obj.getClass())) { - return Arrays.equals(this.locations, ((FileLink) obj).locations); - } - - return false; - } - - @Override - public int hashCode() { - return Arrays.hashCode(locations); - } -} - diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java deleted file mode 100644 index 5128662..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ /dev/null @@ -1,528 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.io; - -import java.io.IOException; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.HFileArchiveUtil; -import org.apache.hadoop.hbase.util.Pair; - -/** - * HFileLink describes a link to an hfile. - * - * An hfile can be served from a region or from the hfile archive directory (/hbase/.archive) - * HFileLink allows to access the referenced hfile regardless of the location where it is. - * - *

Searches for hfiles in the following order and locations: - *

    - *
  • /hbase/table/region/cf/hfile
  • - *
  • /hbase/.archive/table/region/cf/hfile
  • - *
- * - * The link checks first in the original path if it is not present - * it fallbacks to the archived path. - */ -@InterfaceAudience.Private -@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="EQ_DOESNT_OVERRIDE_EQUALS", - justification="To be fixed but warning suppressed for now") -public class HFileLink extends FileLink { - private static final Log LOG = LogFactory.getLog(HFileLink.class); - - /** - * A non-capture group, for HFileLink, so that this can be embedded. - * The HFileLink describe a link to an hfile in a different table/region - * and the name is in the form: table=region-hfile. - *

- * Table name is ([a-zA-Z_0-9][a-zA-Z_0-9.-]*), so '=' is an invalid character for the table name. - * Region name is ([a-f0-9]+), so '-' is an invalid character for the region name. - * HFile is ([0-9a-f]+(?:_SeqId_[0-9]+_)?) covering the plain hfiles (uuid) - * and the bulk loaded (_SeqId_[0-9]+_) hfiles. - */ - public static final String LINK_NAME_REGEX = - String.format("(?:(?:%s=)?)%s=%s-%s", - TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, - HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFileInfo.HFILE_NAME_REGEX); - - /** Define the HFile Link name parser in the form of: table=region-hfile */ - //made package private for testing - static final Pattern LINK_NAME_PATTERN = - Pattern.compile(String.format("^(?:(%s)(?:\\=))?(%s)=(%s)-(%s)$", - TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, - HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFileInfo.HFILE_NAME_REGEX)); - - /** - * The pattern should be used for hfile and reference links - * that can be found in /hbase/table/region/family/ - */ - private static final Pattern REF_OR_HFILE_LINK_PATTERN = - Pattern.compile(String.format("^(?:(%s)(?:=))?(%s)=(%s)-(.+)$", - TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, - HRegionInfo.ENCODED_REGION_NAME_REGEX)); - - private final Path archivePath; - private final Path originPath; - private final Path mobPath; - private final Path tempPath; - - /** - * Dead simple hfile link constructor - */ - public HFileLink(final Path originPath, final Path tempPath, final Path mobPath, - final Path archivePath) { - this.tempPath = tempPath; - this.originPath = originPath; - this.mobPath = mobPath; - this.archivePath = archivePath; - setLocations(originPath, tempPath, mobPath, archivePath); - } - - - /** - * @param conf {@link Configuration} from which to extract specific archive locations - * @param hFileLinkPattern The path ending with a HFileLink pattern. (table=region-hfile) - * @throws IOException on unexpected error. - */ - public static final HFileLink buildFromHFileLinkPattern(Configuration conf, Path hFileLinkPattern) - throws IOException { - return buildFromHFileLinkPattern(FSUtils.getRootDir(conf), - HFileArchiveUtil.getArchivePath(conf), hFileLinkPattern); - } - - - - /** - * @param rootDir Path to the root directory where hbase files are stored - * @param archiveDir Path to the hbase archive directory - * @param hFileLinkPattern The path of the HFile Link. - */ - public final static HFileLink buildFromHFileLinkPattern(final Path rootDir, - final Path archiveDir, - final Path hFileLinkPattern) { - Path hfilePath = getHFileLinkPatternRelativePath(hFileLinkPattern); - Path tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath); - Path originPath = new Path(rootDir, hfilePath); - Path mobPath = new Path(new Path(rootDir, MobConstants.MOB_DIR_NAME), hfilePath); - Path archivePath = new Path(archiveDir, hfilePath); - return new HFileLink(originPath, tempPath, mobPath, archivePath); - } - - /** - * Create an HFileLink relative path for the table/region/family/hfile location - * @param table Table name - * @param region Region Name - * @param family Family Name - * @param hfile HFile Name - * @return the relative Path to open the specified table/region/family/hfile link - */ - public static Path createPath(final TableName table, final String region, - final String family, final String hfile) { - if (HFileLink.isHFileLink(hfile)) { - return new Path(family, hfile); - } - return new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); - } - - /** - * Create an HFileLink instance from table/region/family/hfile location - * @param conf {@link Configuration} from which to extract specific archive locations - * @param table Table name - * @param region Region Name - * @param family Family Name - * @param hfile HFile Name - * @return Link to the file with the specified table/region/family/hfile location - * @throws IOException on unexpected error. - */ - public static HFileLink build(final Configuration conf, final TableName table, - final String region, final String family, final String hfile) - throws IOException { - return HFileLink.buildFromHFileLinkPattern(conf, createPath(table, region, family, hfile)); - } - - /** - * @return the origin path of the hfile. - */ - public Path getOriginPath() { - return this.originPath; - } - - /** - * @return the path of the archived hfile. - */ - public Path getArchivePath() { - return this.archivePath; - } - - /** - * @return the path of the mob hfiles. - */ - public Path getMobPath() { - return this.mobPath; - } - - /** - * @param path Path to check. - * @return True if the path is a HFileLink. - */ - public static boolean isHFileLink(final Path path) { - return isHFileLink(path.getName()); - } - - - /** - * @param fileName File name to check. - * @return True if the path is a HFileLink. - */ - public static boolean isHFileLink(String fileName) { - Matcher m = LINK_NAME_PATTERN.matcher(fileName); - if (!m.matches()) return false; - return m.groupCount() > 2 && m.group(4) != null && m.group(3) != null && m.group(2) != null; - } - - /** - * Convert a HFileLink path to a table relative path. - * e.g. the link: /hbase/test/0123/cf/testtb=4567-abcd - * becomes: /hbase/testtb/4567/cf/abcd - * - * @param path HFileLink path - * @return Relative table path - * @throws IOException on unexpected error. - */ - private static Path getHFileLinkPatternRelativePath(final Path path) { - // table=region-hfile - Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(path.getName()); - if (!m.matches()) { - throw new IllegalArgumentException(path.getName() + " is not a valid HFileLink pattern!"); - } - - // Convert the HFileLink name into a real table/region/cf/hfile path. - TableName tableName = TableName.valueOf(m.group(1), m.group(2)); - String regionName = m.group(3); - String hfileName = m.group(4); - String familyName = path.getParent().getName(); - Path tableDir = FSUtils.getTableDir(new Path("./"), tableName); - return new Path(tableDir, new Path(regionName, new Path(familyName, - hfileName))); - } - - /** - * Get the HFile name of the referenced link - * - * @param fileName HFileLink file name - * @return the name of the referenced HFile - */ - public static String getReferencedHFileName(final String fileName) { - Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName); - if (!m.matches()) { - throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!"); - } - return(m.group(4)); - } - - /** - * Get the Region name of the referenced link - * - * @param fileName HFileLink file name - * @return the name of the referenced Region - */ - public static String getReferencedRegionName(final String fileName) { - Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName); - if (!m.matches()) { - throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!"); - } - return(m.group(3)); - } - - /** - * Get the Table name of the referenced link - * - * @param fileName HFileLink file name - * @return the name of the referenced Table - */ - public static TableName getReferencedTableName(final String fileName) { - Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName); - if (!m.matches()) { - throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!"); - } - return(TableName.valueOf(m.group(1), m.group(2))); - } - - /** - * Create a new HFileLink name - * - * @param hfileRegionInfo - Linked HFile Region Info - * @param hfileName - Linked HFile name - * @return file name of the HFile Link - */ - public static String createHFileLinkName(final HRegionInfo hfileRegionInfo, - final String hfileName) { - return createHFileLinkName(hfileRegionInfo.getTable(), - hfileRegionInfo.getEncodedName(), hfileName); - } - - /** - * Create a new HFileLink name - * - * @param tableName - Linked HFile table name - * @param regionName - Linked HFile region name - * @param hfileName - Linked HFile name - * @return file name of the HFile Link - */ - public static String createHFileLinkName(final TableName tableName, - final String regionName, final String hfileName) { - String s = String.format("%s=%s-%s", - tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '='), - regionName, hfileName); - return s; - } - - /** - * Create a new HFileLink - * - *

It also adds a back-reference to the hfile back-reference directory - * to simplify the reference-count and the cleaning process. - * - * @param conf {@link Configuration} to read for the archive directory name - * @param fs {@link FileSystem} on which to write the HFileLink - * @param dstFamilyPath - Destination path (table/region/cf/) - * @param hfileRegionInfo - Linked HFile Region Info - * @param hfileName - Linked HFile name - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure - */ - public static boolean create(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, - final String hfileName) throws IOException { - return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true); - } - - /** - * Create a new HFileLink - * - *

It also adds a back-reference to the hfile back-reference directory - * to simplify the reference-count and the cleaning process. - * - * @param conf {@link Configuration} to read for the archive directory name - * @param fs {@link FileSystem} on which to write the HFileLink - * @param dstFamilyPath - Destination path (table/region/cf/) - * @param hfileRegionInfo - Linked HFile Region Info - * @param hfileName - Linked HFile name - * @param createBackRef - Whether back reference should be created. Defaults to true. - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure - */ - public static boolean create(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, - final String hfileName, final boolean createBackRef) throws IOException { - TableName linkedTable = hfileRegionInfo.getTable(); - String linkedRegion = hfileRegionInfo.getEncodedName(); - return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef); - } - - /** - * Create a new HFileLink - * - *

It also adds a back-reference to the hfile back-reference directory - * to simplify the reference-count and the cleaning process. - * - * @param conf {@link Configuration} to read for the archive directory name - * @param fs {@link FileSystem} on which to write the HFileLink - * @param dstFamilyPath - Destination path (table/region/cf/) - * @param linkedTable - Linked Table Name - * @param linkedRegion - Linked Region Name - * @param hfileName - Linked HFile name - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure - */ - public static boolean create(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, - final String hfileName) throws IOException { - return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true); - } - - /** - * Create a new HFileLink - * - *

It also adds a back-reference to the hfile back-reference directory - * to simplify the reference-count and the cleaning process. - * - * @param conf {@link Configuration} to read for the archive directory name - * @param fs {@link FileSystem} on which to write the HFileLink - * @param dstFamilyPath - Destination path (table/region/cf/) - * @param linkedTable - Linked Table Name - * @param linkedRegion - Linked Region Name - * @param hfileName - Linked HFile name - * @param createBackRef - Whether back reference should be created. Defaults to true. - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure - */ - public static boolean create(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, - final String hfileName, final boolean createBackRef) throws IOException { - String familyName = dstFamilyPath.getName(); - String regionName = dstFamilyPath.getParent().getName(); - String tableName = FSUtils.getTableName(dstFamilyPath.getParent().getParent()) - .getNameAsString(); - - String name = createHFileLinkName(linkedTable, linkedRegion, hfileName); - String refName = createBackReferenceName(tableName, regionName); - - // Make sure the destination directory exists - fs.mkdirs(dstFamilyPath); - - // Make sure the FileLink reference directory exists - Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, - linkedTable, linkedRegion, familyName); - Path backRefPath = null; - if (createBackRef) { - Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName); - fs.mkdirs(backRefssDir); - - // Create the reference for the link - backRefPath = new Path(backRefssDir, refName); - fs.createNewFile(backRefPath); - } - try { - // Create the link - return fs.createNewFile(new Path(dstFamilyPath, name)); - } catch (IOException e) { - LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e); - // Revert the reference if the link creation failed - if (createBackRef) { - fs.delete(backRefPath, false); - } - throw e; - } - } - - /** - * Create a new HFileLink starting from a hfileLink name - * - *

It also adds a back-reference to the hfile back-reference directory - * to simplify the reference-count and the cleaning process. - * - * @param conf {@link Configuration} to read for the archive directory name - * @param fs {@link FileSystem} on which to write the HFileLink - * @param dstFamilyPath - Destination path (table/region/cf/) - * @param hfileLinkName - HFileLink name (it contains hfile-region-table) - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure - */ - public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final String hfileLinkName) - throws IOException { - return createFromHFileLink(conf, fs, dstFamilyPath, hfileLinkName, true); - } - - /** - * Create a new HFileLink starting from a hfileLink name - * - *

It also adds a back-reference to the hfile back-reference directory - * to simplify the reference-count and the cleaning process. - * - * @param conf {@link Configuration} to read for the archive directory name - * @param fs {@link FileSystem} on which to write the HFileLink - * @param dstFamilyPath - Destination path (table/region/cf/) - * @param hfileLinkName - HFileLink name (it contains hfile-region-table) - * @param createBackRef - Whether back reference should be created. Defaults to true. - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure - */ - public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef) - throws IOException { - Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName); - if (!m.matches()) { - throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!"); - } - return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)), - m.group(3), m.group(4), createBackRef); - } - - /** - * Create the back reference name - */ - //package-private for testing - static String createBackReferenceName(final String tableNameStr, - final String regionName) { - - return regionName + "." + tableNameStr.replace(TableName.NAMESPACE_DELIM, '='); - } - - /** - * Get the full path of the HFile referenced by the back reference - * - * @param rootDir root hbase directory - * @param linkRefPath Link Back Reference path - * @return full path of the referenced hfile - */ - public static Path getHFileFromBackReference(final Path rootDir, final Path linkRefPath) { - Pair p = parseBackReferenceName(linkRefPath.getName()); - TableName linkTableName = p.getFirst(); - String linkRegionName = p.getSecond(); - - String hfileName = getBackReferenceFileName(linkRefPath.getParent()); - Path familyPath = linkRefPath.getParent().getParent(); - Path regionPath = familyPath.getParent(); - Path tablePath = regionPath.getParent(); - - String linkName = createHFileLinkName(FSUtils.getTableName(tablePath), - regionPath.getName(), hfileName); - Path linkTableDir = FSUtils.getTableDir(rootDir, linkTableName); - Path regionDir = HRegion.getRegionDir(linkTableDir, linkRegionName); - return new Path(new Path(regionDir, familyPath.getName()), linkName); - } - - static Pair parseBackReferenceName(String name) { - int separatorIndex = name.indexOf('.'); - String linkRegionName = name.substring(0, separatorIndex); - String tableSubstr = name.substring(separatorIndex + 1) - .replace('=', TableName.NAMESPACE_DELIM); - TableName linkTableName = TableName.valueOf(tableSubstr); - return new Pair(linkTableName, linkRegionName); - } - - /** - * Get the full path of the HFile referenced by the back reference - * - * @param conf {@link Configuration} to read for the archive directory name - * @param linkRefPath Link Back Reference path - * @return full path of the referenced hfile - * @throws IOException on unexpected error. - */ - public static Path getHFileFromBackReference(final Configuration conf, final Path linkRefPath) - throws IOException { - return getHFileFromBackReference(FSUtils.getRootDir(conf), linkRefPath); - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/WALLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/WALLink.java index 344d496..359a150 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/WALLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/WALLink.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.fs.legacy.io.FileLink; import org.apache.hadoop.hbase.util.FSUtils; /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java index c831efd..5dde820 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java @@ -72,7 +72,7 @@ import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory; import org.apache.hadoop.hbase.client.SecureBulkLoadClient; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.HalfStoreFileReader; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java index ecd2415..d7649aa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java @@ -55,7 +55,7 @@ import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.backup.HFileArchiver; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java index 29b7e8a..0dcdf74 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java @@ -52,7 +52,7 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java index 79cd39c..f14013b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java @@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java index b6b4f67..f58700a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.InvalidFamilyOperationException; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; @@ -49,7 +48,7 @@ import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.BufferedMutatorParams; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.mob.MobConstants; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java index d022c99..b1fb313 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.HalfStoreFileReader; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.hfile.CacheConfig; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index 9734f43..01548cc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -49,8 +49,8 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.io.FileLink; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.FileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.WALLink; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.mob.MobUtils; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index 2d9d3d7..8c24b1e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -49,7 +49,7 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.fs.RegionStorage; import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.monitoring.MonitoredTask; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java index 8050d93..6f7fd8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java @@ -50,7 +50,7 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.WALLink; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java index c655dd8..e97a60b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java @@ -38,7 +38,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java index 0655a0f..31e8d62 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileLinkCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileLinkCleaner.java index 998481a..dfa8fb5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileLinkCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/cleaner/TestHFileLinkCleaner.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ClusterConnection; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.FSUtils; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/io/TestFileLink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/io/TestFileLink.java new file mode 100644 index 0000000..4acdaf2 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/io/TestFileLink.java @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.fs.legacy.io.FileLink; +import org.apache.hadoop.hbase.testclassification.IOTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test that FileLink switches between alternate locations + * when the current location moves or gets deleted. + */ +@Category({IOTests.class, MediumTests.class}) +public class TestFileLink { + + @Test + public void testEquals() { + Path p1 = new Path("/p1"); + Path p2 = new Path("/p2"); + Path p3 = new Path("/p3"); + + assertEquals(new FileLink(), new FileLink()); + assertEquals(new FileLink(p1), new FileLink(p1)); + assertEquals(new FileLink(p1, p2), new FileLink(p1, p2)); + assertEquals(new FileLink(p1, p2, p3), new FileLink(p1, p2, p3)); + + assertNotEquals(new FileLink(p1), new FileLink(p3)); + assertNotEquals(new FileLink(p1, p2), new FileLink(p1)); + assertNotEquals(new FileLink(p1, p2), new FileLink(p2)); + assertNotEquals(new FileLink(p1, p2), new FileLink(p2, p1)); // ordering important! + } + + @Test + public void testHashCode() { + Path p1 = new Path("/p1"); + Path p2 = new Path("/p2"); + Path p3 = new Path("/p3"); + + assertEquals(new FileLink().hashCode(), new FileLink().hashCode()); + assertEquals(new FileLink(p1).hashCode(), new FileLink(p1).hashCode()); + assertEquals(new FileLink(p1, p2).hashCode(), new FileLink(p1, p2).hashCode()); + assertEquals(new FileLink(p1, p2, p3).hashCode(), new FileLink(p1, p2, p3).hashCode()); + + assertNotEquals(new FileLink(p1).hashCode(), new FileLink(p3).hashCode()); + assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p1).hashCode()); + assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p2).hashCode()); + assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p2, p1).hashCode()); // ordering + } + + /** + * Test, on HDFS, that the FileLink is still readable + * even when the current file gets renamed. + */ + @Test + public void testHDFSLinkReadDuringRename() throws Exception { + HBaseTestingUtility testUtil = new HBaseTestingUtility(); + Configuration conf = testUtil.getConfiguration(); + conf.setInt("dfs.blocksize", 1024 * 1024); + conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024); + + testUtil.startMiniDFSCluster(1); + MiniDFSCluster cluster = testUtil.getDFSCluster(); + FileSystem fs = cluster.getFileSystem(); + assertEquals("hdfs", fs.getUri().getScheme()); + + try { + testLinkReadDuringRename(fs, testUtil.getDefaultRootDirPath()); + } finally { + testUtil.shutdownMiniCluster(); + } + } + + /** + * Test, on a local filesystem, that the FileLink is still readable + * even when the current file gets renamed. + */ + @Test + public void testLocalLinkReadDuringRename() throws IOException { + HBaseTestingUtility testUtil = new HBaseTestingUtility(); + FileSystem fs = testUtil.getTestFileSystem(); + assertEquals("file", fs.getUri().getScheme()); + testLinkReadDuringRename(fs, testUtil.getDataTestDir()); + } + + /** + * Test that link is still readable even when the current file gets renamed. + */ + private void testLinkReadDuringRename(FileSystem fs, Path rootDir) throws IOException { + Path originalPath = new Path(rootDir, "test.file"); + Path archivedPath = new Path(rootDir, "archived.file"); + + writeSomeData(fs, originalPath, 256 << 20, (byte)2); + + List files = new ArrayList(); + files.add(originalPath); + files.add(archivedPath); + + FileLink link = new FileLink(files); + FSDataInputStream in = link.open(fs); + try { + byte[] data = new byte[8192]; + long size = 0; + + // Read from origin + int n = in.read(data); + dataVerify(data, n, (byte)2); + size += n; + + if (FSUtils.WINDOWS) { + in.close(); + } + + // Move origin to archive + assertFalse(fs.exists(archivedPath)); + fs.rename(originalPath, archivedPath); + assertFalse(fs.exists(originalPath)); + assertTrue(fs.exists(archivedPath)); + + if (FSUtils.WINDOWS) { + in = link.open(fs); // re-read from beginning + in.read(data); + } + + // Try to read to the end + while ((n = in.read(data)) > 0) { + dataVerify(data, n, (byte)2); + size += n; + } + + assertEquals(256 << 20, size); + } finally { + in.close(); + if (fs.exists(originalPath)) fs.delete(originalPath, true); + if (fs.exists(archivedPath)) fs.delete(archivedPath, true); + } + } + + /** + * Test that link is still readable even when the current file gets deleted. + * + * NOTE: This test is valid only on HDFS. + * When a file is deleted from a local file-system, it is simply 'unlinked'. + * The inode, which contains the file's data, is not deleted until all + * processes have finished with it. + * In HDFS when the request exceed the cached block locations, + * a query to the namenode is performed, using the filename, + * and the deleted file doesn't exists anymore (FileNotFoundException). + */ + @Test + public void testHDFSLinkReadDuringDelete() throws Exception { + HBaseTestingUtility testUtil = new HBaseTestingUtility(); + Configuration conf = testUtil.getConfiguration(); + conf.setInt("dfs.blocksize", 1024 * 1024); + conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024); + + testUtil.startMiniDFSCluster(1); + MiniDFSCluster cluster = testUtil.getDFSCluster(); + FileSystem fs = cluster.getFileSystem(); + assertEquals("hdfs", fs.getUri().getScheme()); + + try { + List files = new ArrayList(); + for (int i = 0; i < 3; i++) { + Path path = new Path(String.format("test-data-%d", i)); + writeSomeData(fs, path, 1 << 20, (byte)i); + files.add(path); + } + + FileLink link = new FileLink(files); + FSDataInputStream in = link.open(fs); + try { + byte[] data = new byte[8192]; + int n; + + // Switch to file 1 + n = in.read(data); + dataVerify(data, n, (byte)0); + fs.delete(files.get(0), true); + skipBuffer(in, (byte)0); + + // Switch to file 2 + n = in.read(data); + dataVerify(data, n, (byte)1); + fs.delete(files.get(1), true); + skipBuffer(in, (byte)1); + + // Switch to file 3 + n = in.read(data); + dataVerify(data, n, (byte)2); + fs.delete(files.get(2), true); + skipBuffer(in, (byte)2); + + // No more files available + try { + n = in.read(data); + assert(n <= 0); + } catch (FileNotFoundException e) { + assertTrue(true); + } + } finally { + in.close(); + } + } finally { + testUtil.shutdownMiniCluster(); + } + } + + /** + * Write up to 'size' bytes with value 'v' into a new file called 'path'. + */ + private void writeSomeData (FileSystem fs, Path path, long size, byte v) throws IOException { + byte[] data = new byte[4096]; + for (int i = 0; i < data.length; i++) { + data[i] = v; + } + + FSDataOutputStream stream = fs.create(path); + try { + long written = 0; + while (written < size) { + stream.write(data, 0, data.length); + written += data.length; + } + } finally { + stream.close(); + } + } + + /** + * Verify that all bytes in 'data' have 'v' as value. + */ + private static void dataVerify(byte[] data, int n, byte v) { + for (int i = 0; i < n; ++i) { + assertEquals(v, data[i]); + } + } + + private static void skipBuffer(FSDataInputStream in, byte v) throws IOException { + byte[] data = new byte[8192]; + try { + int n; + while ((n = in.read(data)) == data.length) { + for (int i = 0; i < data.length; ++i) { + if (data[i] != v) + throw new Exception("File changed"); + } + } + } catch (Exception e) { + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/io/TestHFileLink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/io/TestHFileLink.java new file mode 100644 index 0000000..333abb7 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/legacy/io/TestHFileLink.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.fs.legacy.io; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.fs.legacy.io.FileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; +import org.apache.hadoop.hbase.testclassification.IOTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.regex.Matcher; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test that FileLink switches between alternate locations + * when the current location moves or gets deleted. + */ +@Category({IOTests.class, SmallTests.class}) +public class TestHFileLink { + + @Test + public void testValidLinkNames() { + String validLinkNames[] = {"foo=fefefe-0123456", "ns=foo=abababa-fefefefe"}; + + for(String name : validLinkNames) { + Assert.assertTrue("Failed validating:" + name, name.matches(HFileLink.LINK_NAME_REGEX)); + } + + for(String name : validLinkNames) { + Assert.assertTrue("Failed validating:" + name, HFileLink.isHFileLink(name)); + } + + String testName = "foo=fefefe-0123456"; + Assert.assertEquals(TableName.valueOf("foo"), + HFileLink.getReferencedTableName(testName)); + Assert.assertEquals("fefefe", HFileLink.getReferencedRegionName(testName)); + Assert.assertEquals("0123456", HFileLink.getReferencedHFileName(testName)); + Assert.assertEquals(testName, + HFileLink.createHFileLinkName(TableName.valueOf("foo"), "fefefe", "0123456")); + + testName = "ns=foo=fefefe-0123456"; + Assert.assertEquals(TableName.valueOf("ns", "foo"), + HFileLink.getReferencedTableName(testName)); + Assert.assertEquals("fefefe", HFileLink.getReferencedRegionName(testName)); + Assert.assertEquals("0123456", HFileLink.getReferencedHFileName(testName)); + Assert.assertEquals(testName, + HFileLink.createHFileLinkName(TableName.valueOf("ns", "foo"), "fefefe", "0123456")); + + for(String name : validLinkNames) { + Matcher m = HFileLink.LINK_NAME_PATTERN.matcher(name); + assertTrue(m.matches()); + Assert.assertEquals(HFileLink.getReferencedTableName(name), + TableName.valueOf(m.group(1), m.group(2))); + Assert.assertEquals(HFileLink.getReferencedRegionName(name), + m.group(3)); + Assert.assertEquals(HFileLink.getReferencedHFileName(name), + m.group(4)); + } + } + + @Test + public void testBackReference() { + Path rootDir = new Path("/root"); + Path archiveDir = new Path(rootDir, ".archive"); + String storeFileName = "121212"; + String linkDir = FileLink.BACK_REFERENCES_DIRECTORY_PREFIX + storeFileName; + String encodedRegion = "FEFE"; + String cf = "cf1"; + + TableName refTables[] = {TableName.valueOf("refTable"), + TableName.valueOf("ns", "refTable")}; + + for(TableName refTable : refTables) { + Path refTableDir = FSUtils.getTableDir(archiveDir, refTable); + Path refRegionDir = HRegion.getRegionDir(refTableDir, encodedRegion); + Path refDir = new Path(refRegionDir, cf); + Path refLinkDir = new Path(refDir, linkDir); + String refStoreFileName = refTable.getNameAsString().replace( + TableName.NAMESPACE_DELIM, '=') + "=" + encodedRegion + "-" + storeFileName; + + TableName tableNames[] = {TableName.valueOf("tableName1"), + TableName.valueOf("ns", "tableName2")}; + + for( TableName tableName : tableNames) { + Path tableDir = FSUtils.getTableDir(rootDir, tableName); + Path regionDir = HRegion.getRegionDir(tableDir, encodedRegion); + Path cfDir = new Path(regionDir, cf); + + //Verify back reference creation + assertEquals(encodedRegion+"."+ + tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '='), + HFileLink.createBackReferenceName(tableName.getNameAsString(), + encodedRegion)); + + //verify parsing back reference + Pair parsedRef = + HFileLink.parseBackReferenceName(encodedRegion+"."+ + tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '=')); + assertEquals(parsedRef.getFirst(), tableName); + assertEquals(parsedRef.getSecond(), encodedRegion); + + //verify resolving back reference + Path storeFileDir = new Path(refLinkDir, encodedRegion+"."+ + tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '=')); + Path linkPath = new Path(cfDir, refStoreFileName); + assertEquals(linkPath, HFileLink.getHFileFromBackReference(rootDir, storeFileDir)); + } + } + } + + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java deleted file mode 100644 index 0da685f..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFileLink.java +++ /dev/null @@ -1,286 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.io; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.testclassification.IOTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Test that FileLink switches between alternate locations - * when the current location moves or gets deleted. - */ -@Category({IOTests.class, MediumTests.class}) -public class TestFileLink { - - @Test - public void testEquals() { - Path p1 = new Path("/p1"); - Path p2 = new Path("/p2"); - Path p3 = new Path("/p3"); - - assertEquals(new FileLink(), new FileLink()); - assertEquals(new FileLink(p1), new FileLink(p1)); - assertEquals(new FileLink(p1, p2), new FileLink(p1, p2)); - assertEquals(new FileLink(p1, p2, p3), new FileLink(p1, p2, p3)); - - assertNotEquals(new FileLink(p1), new FileLink(p3)); - assertNotEquals(new FileLink(p1, p2), new FileLink(p1)); - assertNotEquals(new FileLink(p1, p2), new FileLink(p2)); - assertNotEquals(new FileLink(p1, p2), new FileLink(p2, p1)); // ordering important! - } - - @Test - public void testHashCode() { - Path p1 = new Path("/p1"); - Path p2 = new Path("/p2"); - Path p3 = new Path("/p3"); - - assertEquals(new FileLink().hashCode(), new FileLink().hashCode()); - assertEquals(new FileLink(p1).hashCode(), new FileLink(p1).hashCode()); - assertEquals(new FileLink(p1, p2).hashCode(), new FileLink(p1, p2).hashCode()); - assertEquals(new FileLink(p1, p2, p3).hashCode(), new FileLink(p1, p2, p3).hashCode()); - - assertNotEquals(new FileLink(p1).hashCode(), new FileLink(p3).hashCode()); - assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p1).hashCode()); - assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p2).hashCode()); - assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p2, p1).hashCode()); // ordering - } - - /** - * Test, on HDFS, that the FileLink is still readable - * even when the current file gets renamed. - */ - @Test - public void testHDFSLinkReadDuringRename() throws Exception { - HBaseTestingUtility testUtil = new HBaseTestingUtility(); - Configuration conf = testUtil.getConfiguration(); - conf.setInt("dfs.blocksize", 1024 * 1024); - conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024); - - testUtil.startMiniDFSCluster(1); - MiniDFSCluster cluster = testUtil.getDFSCluster(); - FileSystem fs = cluster.getFileSystem(); - assertEquals("hdfs", fs.getUri().getScheme()); - - try { - testLinkReadDuringRename(fs, testUtil.getDefaultRootDirPath()); - } finally { - testUtil.shutdownMiniCluster(); - } - } - - /** - * Test, on a local filesystem, that the FileLink is still readable - * even when the current file gets renamed. - */ - @Test - public void testLocalLinkReadDuringRename() throws IOException { - HBaseTestingUtility testUtil = new HBaseTestingUtility(); - FileSystem fs = testUtil.getTestFileSystem(); - assertEquals("file", fs.getUri().getScheme()); - testLinkReadDuringRename(fs, testUtil.getDataTestDir()); - } - - /** - * Test that link is still readable even when the current file gets renamed. - */ - private void testLinkReadDuringRename(FileSystem fs, Path rootDir) throws IOException { - Path originalPath = new Path(rootDir, "test.file"); - Path archivedPath = new Path(rootDir, "archived.file"); - - writeSomeData(fs, originalPath, 256 << 20, (byte)2); - - List files = new ArrayList(); - files.add(originalPath); - files.add(archivedPath); - - FileLink link = new FileLink(files); - FSDataInputStream in = link.open(fs); - try { - byte[] data = new byte[8192]; - long size = 0; - - // Read from origin - int n = in.read(data); - dataVerify(data, n, (byte)2); - size += n; - - if (FSUtils.WINDOWS) { - in.close(); - } - - // Move origin to archive - assertFalse(fs.exists(archivedPath)); - fs.rename(originalPath, archivedPath); - assertFalse(fs.exists(originalPath)); - assertTrue(fs.exists(archivedPath)); - - if (FSUtils.WINDOWS) { - in = link.open(fs); // re-read from beginning - in.read(data); - } - - // Try to read to the end - while ((n = in.read(data)) > 0) { - dataVerify(data, n, (byte)2); - size += n; - } - - assertEquals(256 << 20, size); - } finally { - in.close(); - if (fs.exists(originalPath)) fs.delete(originalPath, true); - if (fs.exists(archivedPath)) fs.delete(archivedPath, true); - } - } - - /** - * Test that link is still readable even when the current file gets deleted. - * - * NOTE: This test is valid only on HDFS. - * When a file is deleted from a local file-system, it is simply 'unlinked'. - * The inode, which contains the file's data, is not deleted until all - * processes have finished with it. - * In HDFS when the request exceed the cached block locations, - * a query to the namenode is performed, using the filename, - * and the deleted file doesn't exists anymore (FileNotFoundException). - */ - @Test - public void testHDFSLinkReadDuringDelete() throws Exception { - HBaseTestingUtility testUtil = new HBaseTestingUtility(); - Configuration conf = testUtil.getConfiguration(); - conf.setInt("dfs.blocksize", 1024 * 1024); - conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024); - - testUtil.startMiniDFSCluster(1); - MiniDFSCluster cluster = testUtil.getDFSCluster(); - FileSystem fs = cluster.getFileSystem(); - assertEquals("hdfs", fs.getUri().getScheme()); - - try { - List files = new ArrayList(); - for (int i = 0; i < 3; i++) { - Path path = new Path(String.format("test-data-%d", i)); - writeSomeData(fs, path, 1 << 20, (byte)i); - files.add(path); - } - - FileLink link = new FileLink(files); - FSDataInputStream in = link.open(fs); - try { - byte[] data = new byte[8192]; - int n; - - // Switch to file 1 - n = in.read(data); - dataVerify(data, n, (byte)0); - fs.delete(files.get(0), true); - skipBuffer(in, (byte)0); - - // Switch to file 2 - n = in.read(data); - dataVerify(data, n, (byte)1); - fs.delete(files.get(1), true); - skipBuffer(in, (byte)1); - - // Switch to file 3 - n = in.read(data); - dataVerify(data, n, (byte)2); - fs.delete(files.get(2), true); - skipBuffer(in, (byte)2); - - // No more files available - try { - n = in.read(data); - assert(n <= 0); - } catch (FileNotFoundException e) { - assertTrue(true); - } - } finally { - in.close(); - } - } finally { - testUtil.shutdownMiniCluster(); - } - } - - /** - * Write up to 'size' bytes with value 'v' into a new file called 'path'. - */ - private void writeSomeData (FileSystem fs, Path path, long size, byte v) throws IOException { - byte[] data = new byte[4096]; - for (int i = 0; i < data.length; i++) { - data[i] = v; - } - - FSDataOutputStream stream = fs.create(path); - try { - long written = 0; - while (written < size) { - stream.write(data, 0, data.length); - written += data.length; - } - } finally { - stream.close(); - } - } - - /** - * Verify that all bytes in 'data' have 'v' as value. - */ - private static void dataVerify(byte[] data, int n, byte v) { - for (int i = 0; i < n; ++i) { - assertEquals(v, data[i]); - } - } - - private static void skipBuffer(FSDataInputStream in, byte v) throws IOException { - byte[] data = new byte[8192]; - try { - int n; - while ((n = in.read(data)) == data.length) { - for (int i = 0; i < data.length; ++i) { - if (data[i] != v) - throw new Exception("File changed"); - } - } - } catch (Exception e) { - } - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java deleted file mode 100644 index 3a9b3ca..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.io; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.testclassification.IOTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import java.util.regex.Matcher; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test that FileLink switches between alternate locations - * when the current location moves or gets deleted. - */ -@Category({IOTests.class, SmallTests.class}) -public class TestHFileLink { - - @Test - public void testValidLinkNames() { - String validLinkNames[] = {"foo=fefefe-0123456", "ns=foo=abababa-fefefefe"}; - - for(String name : validLinkNames) { - Assert.assertTrue("Failed validating:" + name, name.matches(HFileLink.LINK_NAME_REGEX)); - } - - for(String name : validLinkNames) { - Assert.assertTrue("Failed validating:" + name, HFileLink.isHFileLink(name)); - } - - String testName = "foo=fefefe-0123456"; - Assert.assertEquals(TableName.valueOf("foo"), - HFileLink.getReferencedTableName(testName)); - Assert.assertEquals("fefefe", HFileLink.getReferencedRegionName(testName)); - Assert.assertEquals("0123456", HFileLink.getReferencedHFileName(testName)); - Assert.assertEquals(testName, - HFileLink.createHFileLinkName(TableName.valueOf("foo"), "fefefe", "0123456")); - - testName = "ns=foo=fefefe-0123456"; - Assert.assertEquals(TableName.valueOf("ns", "foo"), - HFileLink.getReferencedTableName(testName)); - Assert.assertEquals("fefefe", HFileLink.getReferencedRegionName(testName)); - Assert.assertEquals("0123456", HFileLink.getReferencedHFileName(testName)); - Assert.assertEquals(testName, - HFileLink.createHFileLinkName(TableName.valueOf("ns", "foo"), "fefefe", "0123456")); - - for(String name : validLinkNames) { - Matcher m = HFileLink.LINK_NAME_PATTERN.matcher(name); - assertTrue(m.matches()); - Assert.assertEquals(HFileLink.getReferencedTableName(name), - TableName.valueOf(m.group(1), m.group(2))); - Assert.assertEquals(HFileLink.getReferencedRegionName(name), - m.group(3)); - Assert.assertEquals(HFileLink.getReferencedHFileName(name), - m.group(4)); - } - } - - @Test - public void testBackReference() { - Path rootDir = new Path("/root"); - Path archiveDir = new Path(rootDir, ".archive"); - String storeFileName = "121212"; - String linkDir = FileLink.BACK_REFERENCES_DIRECTORY_PREFIX + storeFileName; - String encodedRegion = "FEFE"; - String cf = "cf1"; - - TableName refTables[] = {TableName.valueOf("refTable"), - TableName.valueOf("ns", "refTable")}; - - for(TableName refTable : refTables) { - Path refTableDir = FSUtils.getTableDir(archiveDir, refTable); - Path refRegionDir = HRegion.getRegionDir(refTableDir, encodedRegion); - Path refDir = new Path(refRegionDir, cf); - Path refLinkDir = new Path(refDir, linkDir); - String refStoreFileName = refTable.getNameAsString().replace( - TableName.NAMESPACE_DELIM, '=') + "=" + encodedRegion + "-" + storeFileName; - - TableName tableNames[] = {TableName.valueOf("tableName1"), - TableName.valueOf("ns", "tableName2")}; - - for( TableName tableName : tableNames) { - Path tableDir = FSUtils.getTableDir(rootDir, tableName); - Path regionDir = HRegion.getRegionDir(tableDir, encodedRegion); - Path cfDir = new Path(regionDir, cf); - - //Verify back reference creation - assertEquals(encodedRegion+"."+ - tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '='), - HFileLink.createBackReferenceName(tableName.getNameAsString(), - encodedRegion)); - - //verify parsing back reference - Pair parsedRef = - HFileLink.parseBackReferenceName(encodedRegion+"."+ - tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '=')); - assertEquals(parsedRef.getFirst(), tableName); - assertEquals(parsedRef.getSecond(), encodedRegion); - - //verify resolving back reference - Path storeFileDir = new Path(refLinkDir, encodedRegion+"."+ - tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '=')); - Path linkPath = new Path(cfDir, refStoreFileName); - assertEquals(linkPath, HFileLink.getHFileFromBackReference(rootDir, storeFileDir)); - } - } - } - - -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java index 2a97da5..2ad8290 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.fs.RegionStorage.StoreFileVisitor; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileLink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileLink.java index 4ddba13..2c8e101 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileLink.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileLink.java @@ -25,7 +25,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index 863a855..a0736c3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -63,7 +63,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; import org.apache.hadoop.hbase.io.crypto.aes.AES; import org.apache.hadoop.hbase.io.hfile.CacheConfig; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java index 540f6b1..920e04b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -42,10 +42,8 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.fs.RegionStorage; -import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.CacheConfig; @@ -60,7 +58,6 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; -import org.apache.hadoop.hbase.util.FSUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java index 349ec1c..6e24617 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index 40bd961..2fca12c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -41,7 +41,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; @@ -56,9 +55,8 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.fs.MasterStorage; -import org.apache.hadoop.hbase.fs.RegionStorage; import org.apache.hadoop.hbase.fs.legacy.LegacyTableDescriptor; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java index 4011af5..10e820a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; -import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.fs.legacy.io.HFileLink; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -- 2.7.4 (Apple Git-66)