Index: shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java =================================================================== --- shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (revision 1524878) +++ shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (working copy) @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.ProxyFileSystem; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; @@ -783,4 +784,8 @@ public WebHCatJTShim getWebHCatShim(Configuration conf, UserGroupInformation ugi) throws IOException { throw new UnsupportedOperationException("WebHCat does not support Hadoop 0.20.x"); } + @Override + public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { + return new ProxyFileSystem(fs, uri); + } } Index: shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java =================================================================== --- shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (revision 1524878) +++ shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (working copy) @@ -22,6 +22,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.Iterator; +import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; @@ -30,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.ProxyFileSystem; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapred.JobTracker; @@ -372,4 +374,8 @@ return fs.getFileBlockLocations(status, 0, status.getLen()); } + @Override + public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { + return new ProxyFileSystem(fs, uri); + } } Index: shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java =================================================================== --- shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (revision 1524878) +++ shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (working copy) @@ -24,6 +24,8 @@ import java.net.URL; import java.util.Iterator; import java.util.Map; +import java.net.URI; +import java.io.FileNotFoundException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -33,7 +35,10 @@ import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.ProxyFileSystem; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapred.MiniMRCluster; @@ -407,4 +412,40 @@ } } + class ProxyFileSystem23 extends ProxyFileSystem { + public ProxyFileSystem23(FileSystem fs) { + super(fs); + } + public ProxyFileSystem23(FileSystem fs, URI uri) { + super(fs, uri); + } + + @Override + public RemoteIterator listLocatedStatus(final Path f) + throws FileNotFoundException, IOException { + return new RemoteIterator() { + private RemoteIterator stats = + ProxyFileSystem23.super.listLocatedStatus( + ProxyFileSystem23.super.swizzleParamPath(f)); + + @Override + public boolean hasNext() throws IOException { + return stats.hasNext(); + } + + @Override + public LocatedFileStatus next() throws IOException { + LocatedFileStatus result = stats.next(); + return new LocatedFileStatus( + ProxyFileSystem23.super.swizzleFileStatus(result, false), + result.getBlockLocations()); + } + }; + } + } + + @Override + public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { + return new ProxyFileSystem23(fs, uri); + } } Index: shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java =================================================================== --- shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (revision 1524878) +++ shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (working copy) @@ -622,4 +622,7 @@ @Override abstract public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf) throws IOException; + + @Override + abstract public FileSystem createProxyFileSystem(FileSystem fs, URI uri); } Index: shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java =================================================================== --- shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java (revision 1524878) +++ shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java (working copy) @@ -45,7 +45,7 @@ - private Path swizzleParamPath(Path p) { + protected Path swizzleParamPath(Path p) { String pathUriString = p.toUri().toString(); URI newPathUri = URI.create(pathUriString); return new Path (realScheme, realAuthority, newPathUri.getPath()); @@ -57,7 +57,7 @@ return new Path (myScheme, myAuthority, newPathUri.getPath()); } - private FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) { + protected FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) { FileStatus ret = new FileStatus(orig.getLen(), orig.isDir(), orig.getReplication(), orig.getBlockSize(), orig.getModificationTime(), Index: shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java =================================================================== --- shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java (revision 1524878) +++ shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java (working copy) @@ -23,6 +23,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.hive.shims.HadoopShims; /**************************************************************** * A Proxy for LocalFileSystem @@ -61,8 +63,10 @@ String authority = name.getAuthority() != null ? name.getAuthority() : ""; String proxyUriString = nameUriString + "://" + authority + "/"; - fs = new ProxyFileSystem(localFs, URI.create(proxyUriString)); + fs = ShimLoader.getHadoopShims().createProxyFileSystem( + localFs, URI.create(proxyUriString)); + fs.initialize(name, conf); } } Index: shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java =================================================================== --- shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (revision 1524878) +++ shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (working copy) @@ -562,4 +562,10 @@ */ public void close(); } + + /** + * Create a proxy file system that can serve a given scheme/authority using some + * other file system. + */ + public FileSystem createProxyFileSystem(FileSystem fs, URI uri); }