Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1056008) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -333,8 +333,9 @@ // Print column names in output HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false), - HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false); - + HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false), + + HIVE_INDEX_IGNORE_HDFS_LOC("hive.index.compact.file.ignore.hdfs", false), ; Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 1056008) +++ conf/hive-default.xml (working copy) @@ -847,5 +847,11 @@ Whether to throw an excpetion if dynamic partition insert generates empty results. + + hive.index.compact.file.ignore.hdfs + false + True the hdfs location stored in the index file will be igbored at runtime. + If the data got moved or the name of the cluster got changed, the index data should still be usable. + Index: ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java (revision 1056008) +++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java (working copy) @@ -44,7 +44,7 @@ for (Path dir : dirs) { PartitionDesc part = HiveFileFormatUtils .getPartitionDescFromPathRecursively(pathToPartitionInfo, dir, - IOPrepareCache.get().allocatePartitionDescMap()); + IOPrepareCache.get().allocatePartitionDescMap(), true); // create a new InputFormat instance if this is the first time to see this // class Class inputFormatClass = part.getInputFileFormatClass(); Index: ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java (revision 1056008) +++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java (working copy) @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; @@ -74,6 +75,7 @@ JobConf job = null; BytesRefWritable[] bytesRef = new BytesRefWritable[2]; + boolean ignoreHdfsLoc = false; public HiveCompactIndexResult(String indexFile, JobConf conf) throws IOException, HiveException { @@ -81,6 +83,7 @@ bytesRef[0] = new BytesRefWritable(); bytesRef[1] = new BytesRefWritable(); + ignoreHdfsLoc = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_INDEX_IGNORE_HDFS_LOC); if (indexFile != null) { Path indexFilePath = new Path(indexFile); @@ -128,6 +131,11 @@ + line.toString()); } String bucketFileName = new String(bytes, 0, firstEnd); + + if (ignoreHdfsLoc) { + Path tmpPath = new Path(bucketFileName); + bucketFileName = tmpPath.toUri().getPath(); + } IBucket bucket = buckets.get(bucketFileName); if (bucket == null) { bucket = new IBucket(bucketFileName); Index: ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (revision 1056008) +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (working copy) @@ -249,11 +249,19 @@ Map pathToPartitionInfo, Path dir, Map, Map> cacheMap) throws IOException { + return getPartitionDescFromPathRecursively(pathToPartitionInfo, dir, + cacheMap, false); + } + + public static PartitionDesc getPartitionDescFromPathRecursively( + Map pathToPartitionInfo, Path dir, + Map, Map> cacheMap, + boolean ignoreSchema) throws IOException { PartitionDesc part = doGetPartitionDescFromPath(pathToPartitionInfo, dir); if (part == null - && (dir.toUri().getScheme() == null || dir.toUri().getScheme().trim() - .equals(""))) { + && (ignoreSchema || (dir.toUri().getScheme() == null || dir.toUri().getScheme().trim() + .equals("")))) { Map newPathToPartitionInfo = null; if (cacheMap != null) { @@ -275,7 +283,7 @@ return part; } else { throw new IOException("cannot find dir = " + dir.toString() - + " in partToPartitionInfo: " + pathToPartitionInfo.keySet()); + + " in pathToPartitionInfo: " + pathToPartitionInfo.keySet()); } }