diff --git src/java/org/apache/hadoop/fs/DF.java src/java/org/apache/hadoop/fs/DF.java index d72797c..8222fc0 100644 --- src/java/org/apache/hadoop/fs/DF.java +++ src/java/org/apache/hadoop/fs/DF.java @@ -19,27 +19,28 @@ package org.apache.hadoop.fs; import java.io.File; import java.io.IOException; -import java.io.BufferedReader; - import java.util.EnumSet; -import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.util.Shell; -/** Filesystem disk space usage statistics. Uses the unix 'df' program. - * Tested on Linux, FreeBSD, Cygwin. */ -public class DF extends Shell { - public static final long DF_INTERVAL_DEFAULT = 3 * 1000; // default DF refresh interval - - private String dirPath; - private String filesystem; - private long capacity; - private long used; - private long available; - private int percentUsed; - private String mount; +/** Filesystem disk space usage statistics. + */ +public abstract class DF { + public static final long DF_INTERVAL_DEFAULT = 3 * 1000; // default DF refresh interval + + /** + * Factory method for basic DF implementation + * @return a DF instance of a default type + */ + public static DF getDF(File path) throws IOException { + // Return a java.io.File-based DF implementation + // that has most of the functionality, and has better performance. + return new JavaDF(path); + } + + public static DF getDF(File path, Configuration conf) throws IOException { + return getDF(path); + } enum OSType { OS_TYPE_UNIX("UNIX"), @@ -60,9 +61,6 @@ public class DF extends Shell { } } - private static final String OS_NAME = System.getProperty("os.name"); - private static final OSType OS_TYPE = getOSType(OS_NAME); - protected static OSType getOSType(String osName) { for (OSType ost : EnumSet.allOf(OSType.class)) { if (ost.match(osName)) { @@ -72,123 +70,14 @@ public class DF extends Shell { return OSType.OS_TYPE_UNIX; } - public DF(File path, Configuration conf) throws IOException { - this(path, conf.getLong(CommonConfigurationKeys.FS_DF_INTERVAL_KEY, DF.DF_INTERVAL_DEFAULT)); - } - - public DF(File path, long dfInterval) throws IOException { - super(dfInterval); - this.dirPath = path.getCanonicalPath(); - } - - protected OSType getOSType() { - return OS_TYPE; - } - /// ACCESSORS - public String getDirPath() { - return dirPath; - } - - public String getFilesystem() throws IOException { - run(); - return filesystem; - } - - public long getCapacity() throws IOException { - run(); - return capacity; - } - - public long getUsed() throws IOException { - run(); - return used; - } - - public long getAvailable() throws IOException { - run(); - return available; - } - - public int getPercentUsed() throws IOException { - run(); - return percentUsed; - } - - public String getMount() throws IOException { - run(); - return mount; - } - - public String toString() { - return - "df -k " + mount +"\n" + - filesystem + "\t" + - capacity / 1024 + "\t" + - used / 1024 + "\t" + - available / 1024 + "\t" + - percentUsed + "%\t" + - mount; - } - - @Override - protected String[] getExecString() { - // ignoring the error since the exit code it enough - return new String[] {"bash","-c","exec 'df' '-k' '" + dirPath - + "' 2>/dev/null"}; - } - - @Override - protected void parseExecResult(BufferedReader lines) throws IOException { - lines.readLine(); // skip headings - - String line = lines.readLine(); - if (line == null) { - throw new IOException( "Expecting a line not the end of stream" ); - } - StringTokenizer tokens = - new StringTokenizer(line, " \t\n\r\f%"); - - this.filesystem = tokens.nextToken(); - if (!tokens.hasMoreTokens()) { // for long filesystem name - line = lines.readLine(); - if (line == null) { - throw new IOException( "Expecting a line not the end of stream" ); - } - tokens = new StringTokenizer(line, " \t\n\r\f%"); - } - - switch(getOSType()) { - case OS_TYPE_AIX: - this.capacity = Long.parseLong(tokens.nextToken()) * 1024; - this.available = Long.parseLong(tokens.nextToken()) * 1024; - this.percentUsed = Integer.parseInt(tokens.nextToken()); - tokens.nextToken(); - tokens.nextToken(); - this.mount = tokens.nextToken(); - this.used = this.capacity - this.available; - break; - - case OS_TYPE_WIN: - case OS_TYPE_SOLARIS: - case OS_TYPE_MAC: - case OS_TYPE_UNIX: - default: - this.capacity = Long.parseLong(tokens.nextToken()) * 1024; - this.used = Long.parseLong(tokens.nextToken()) * 1024; - this.available = Long.parseLong(tokens.nextToken()) * 1024; - this.percentUsed = Integer.parseInt(tokens.nextToken()); - this.mount = tokens.nextToken(); - break; - } - } - - public static void main(String[] args) throws Exception { - String path = "."; - if (args.length > 0) - path = args[0]; - - System.out.println(new DF(new File(path), DF_INTERVAL_DEFAULT).toString()); - } + public abstract String getDirPath(); + public abstract String getFilesystem() throws IOException; + public abstract long getCapacity() throws IOException; + public abstract long getUsed() throws IOException; + public abstract long getAvailable() throws IOException; + public abstract int getPercentUsed() throws IOException; + public abstract String getMount() throws IOException; } + diff --git src/java/org/apache/hadoop/fs/LocalDirAllocator.java src/java/org/apache/hadoop/fs/LocalDirAllocator.java index 1aa4663..cde2dbc 100644 --- src/java/org/apache/hadoop/fs/LocalDirAllocator.java +++ src/java/org/apache/hadoop/fs/LocalDirAllocator.java @@ -226,7 +226,7 @@ public class LocalDirAllocator { try { DiskChecker.checkDir(new File(localDirs[i])); dirs.add(localDirs[i]); - dfList.add(new DF(new File(localDirs[i]), 30000)); + dfList.add(DF.getDF(new File(localDirs[i]))); } catch (DiskErrorException de) { LOG.warn( localDirs[i] + "is not writable\n" + StringUtils.stringifyException(de)); diff --git src/java/org/apache/hadoop/fs/LocalFileSystem.java src/java/org/apache/hadoop/fs/LocalFileSystem.java index 199c773..b3f7496 100644 --- src/java/org/apache/hadoop/fs/LocalFileSystem.java +++ src/java/org/apache/hadoop/fs/LocalFileSystem.java @@ -73,7 +73,7 @@ public class LocalFileSystem extends ChecksumFileSystem { File f = ((RawLocalFileSystem)fs).pathToFile(p).getCanonicalFile(); // find highest writable parent dir of f on the same device - String device = new DF(f, getConf()).getMount(); + String device = new PosixDF(f, getConf()).getMount(); File parent = f.getParentFile(); File dir = null; while (parent!=null && parent.canWrite() && parent.toString().startsWith(device)) { diff --git src/test/core/org/apache/hadoop/fs/TestDFVariations.java src/test/core/org/apache/hadoop/fs/TestDFVariations.java index 3999050..82b88ed 100644 --- src/test/core/org/apache/hadoop/fs/TestDFVariations.java +++ src/test/core/org/apache/hadoop/fs/TestDFVariations.java @@ -23,9 +23,11 @@ import java.io.File; import java.io.IOException; import java.util.EnumSet; +import org.apache.hadoop.conf.Configuration; + public class TestDFVariations extends TestCase { - public static class XXDF extends DF { + public static class XXDF extends PosixDF { private final String osName; public XXDF(String osName) throws IOException { super(new File(System.getProperty("test.build.data","/tmp")), 0L); @@ -59,5 +61,43 @@ public class TestDFVariations extends TestCase { } } + public void testCompareJavaAndPosix() throws Exception { + // Instantiate JavaDF and PosixDF and make sure they're consistent. + Configuration conf = new Configuration(); + String testPath = conf.get("test.build.data", "/tmp"); + File testFile = new File(testPath); + + DF javaDF = new JavaDF(testFile); + DF posixDF = new PosixDF(testFile, conf); + + // Capacity should be static and same for both. + assertEquals(javaDF.getCapacity(), posixDF.getCapacity()); + + // Technically, this test races with other disk I/O activities. + // Assume that in the short window between a java df call and a + // posix DF call, no more than 20 MB of data will be written to + // disk. In practice, these should be exact. + final long threshold = 20000000; + final int thresholdPct = 1; // and there's no more than a 1% discrepancy. + assertTrue("Available space mismatch", + Math.abs(javaDF.getAvailable() - posixDF.getAvailable()) <= threshold); + assertTrue("Used space mismatch", + Math.abs(javaDF.getUsed() - posixDF.getUsed()) <= threshold); + assertTrue("Pct used mismatch", + Math.abs(javaDF.getPercentUsed() - posixDF.getPercentUsed()) <= thresholdPct); + } + + public void testDFFactory() throws Exception { + Configuration conf = new Configuration(); + String testPath = conf.get("test.build.data", "/tmp"); + DF df = DF.getDF(new File(testPath)); + + // Assert that this demonstrates some basic functionality. + assertNotNull(df); + assertTrue(df.getAvailable() > 0); + assertTrue(df.getCapacity() > 0); + assertTrue(df.getUsed() > 0); + } + }