Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 984907) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -418,15 +418,19 @@ private void setArchived(Partition p, Path parentDir, String dirInArchive, String archiveName) throws URISyntaxException { assert(isArchived(p) == false); - Map params = p.getParameters(); URI parentUri = parentDir.toUri(); String parentHost = parentUri.getHost(); + String parentPath = parentUri.getPath(); + if (parentPath.startsWith("/")) { + parentPath = parentPath.substring(1); + } + parentPath = parentPath.replaceAll("[^a-zA-Z0-9]", "-"); String harHost = null; if (parentHost == null) { - harHost = ""; + harHost = parentUri.getScheme() + "--" + "--" + parentPath; } else { - harHost = parentUri.getScheme() + "-" + parentHost; + harHost = parentUri.getScheme() + "--" + parentHost + "--" + parentPath; } // harUri is used to access the partition's files, which are in the archive Index: shims/src/0.20/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java =================================================================== --- shims/src/0.20/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java (revision 982451) +++ shims/src/0.20/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java (working copy) @@ -18,8 +18,12 @@ package org.apache.hadoop.hive.shims; +import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; @@ -63,4 +67,163 @@ } return new ContentSummary(summary[0], summary[1], summary[2]); } + + @Override + public void initialize(URI name, Configuration conf) throws IOException { + URI newUri = getNewHarURI(name, conf); + super.initialize(newUri, conf); + } + + + private URI getNewHarURI(URI rawURI, Configuration conf) throws IOException { + + String host = rawURI.getHost(); + + if (host == null) { + return rawURI; + } + + String[] str = host.split("--"); + + if (str[0] == null) { + return rawURI; + } + + StringBuilder newHostBuilder = new StringBuilder(); + newHostBuilder.append(str[0]); + if (str.length > 1 && (str[1].trim().length() > 0)) { + newHostBuilder.append("-" + str[1]); + } + + URI tmp = null; + try { + tmp = new URI(rawURI.getScheme(), rawURI.getUserInfo(), newHostBuilder + .toString(), rawURI.getPort(), rawURI.getPath(), rawURI.getQuery(), + rawURI.getFragment()); + } catch (URISyntaxException e) { + // do nothing should not happen + } + + System.out.println("In getNewHarURI, rawURI is " + rawURI.toString()); + System.out.println("In getNewHarURI, tmpURI is " + tmp.toString()); + + return tmp; + } + + /** + * liststatus returns the children of a directory + * after looking up the index files. + */ + @Override + public FileStatus[] listStatus(Path f) throws IOException { + FileStatus[] result = super.listStatus(f); + FileStatus[] newResult = new FileStatus[result.length]; + for (int i = 0; i < result.length; i++) { + FileStatus fs = result[i]; + FileStatus newFs = constructNewPath(f, fs); + newResult[i] = newFs; + } + return result; + } + + private FileStatus constructNewPath(Path f, FileStatus fs) throws IOException { + try { + URI oldURI = fs.getPath().toUri(); + String oldAuthority = f.toUri().getAuthority(); + String newPathStr = oldURI.getPath(); + newPathStr = newPathStr.substring(0, newPathStr.indexOf("data.har")); + if (newPathStr.startsWith(File.separator)) { + newPathStr = newPathStr.substring(1); + } + if (newPathStr.endsWith(File.separator)) { + newPathStr = newPathStr.substring(0, newPathStr.length() - 1); + } + newPathStr = newPathStr.replaceAll("[^a-zA-Z0-9]", "-"); + StringBuilder newAuthority = new StringBuilder(); + + if (oldAuthority == null || oldAuthority.trim().equals("")) { + newAuthority.append("--"); + newAuthority.append("--"); + newAuthority.append(newPathStr); + } else { + String[] triple = oldAuthority.split("--"); + + if (triple.length > 2) { + newAuthority.append(oldAuthority); + } else { + if (triple.length > 0) { + newAuthority.append(triple[0]); + } + newAuthority.append("--"); + if (triple.length > 1) { + newAuthority.append(triple[1]); + } + newAuthority.append("--"); + newAuthority.append(newPathStr); + } + } + + URI newURI = new URI(oldURI.getScheme(), newAuthority.toString(), oldURI + .getPath(), oldURI.getQuery(), oldURI.getFragment()); + Path newPath = new Path(newURI.toString()); + FileStatus newFs = new FileStatus(fs.getLen(), fs.isDir(), fs + .getReplication(), fs.getBlockSize(), fs.getModificationTime(), + newPath); + return newFs; + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + public FileStatus getFileStatus(Path f) throws IOException { + System.out.println("In getFileStatus, input path is " + f.toString()); + FileStatus fs = super.getFileStatus(f); + FileStatus newFs = constructNewPath(f, fs); + System.out.println("In getFileStatus, one return path is " + newFs.getPath().toString()); + + return newFs; + } + + @Override + public Path makeQualified(Path path) { + Path p = super.makeQualified(path); + + if (p != null) { + URI oldURI = p.toUri(); + URI newURI = null; + String harAuth = oldURI.getAuthority(); + String[] split = harAuth.split("-"); + StringBuilder sb = new StringBuilder(); + sb.append(split[0]); + // underlying-host:port should be the second part + if (split.length > 1) { + String hostPort = split[1]; + String[] hostPortPair = hostPort.split(":"); + if (hostPortPair.length > 0) { + if (hostPortPair[0] != null && (hostPortPair[0].trim().length() > 0)) { + sb.append("-"); + sb.append(hostPortPair[0]); + } + if (hostPortPair.length > 1 && (hostPortPair[1].trim().length() > 0) + && (Integer.parseInt(hostPortPair[1].trim()) > 0)) { + sb.append(":"); + sb.append(hostPortPair[1]); + } + } + } + String newAuth = sb.toString(); + + try { + newURI = new URI(oldURI.getScheme(), newAuth, oldURI.getPath(), oldURI + .getQuery(), oldURI.getFragment()); + } catch (URISyntaxException ue) { + LOG.error("Error in URI ", ue); + } + return new Path(newURI.toString()); + } else { + return null; + } + } + }