Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1056008)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -333,8 +333,9 @@
// Print column names in output
HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false),
- HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false);
-
+ HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false),
+
+ HIVE_INDEX_IGNORE_HDFS_LOC("hive.index.compact.file.ignore.hdfs", false),
;
Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 1056008)
+++ conf/hive-default.xml (working copy)
@@ -847,5 +847,11 @@
Whether to throw an excpetion if dynamic partition insert generates empty results.
+
+ hive.index.compact.file.ignore.hdfs
+ false
+ True the hdfs location stored in the index file will be igbored at runtime.
+ If the data got moved or the name of the cluster got changed, the index data should still be usable.
+
Index: ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java (revision 1056008)
+++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexInputFormat.java (working copy)
@@ -44,7 +44,7 @@
for (Path dir : dirs) {
PartitionDesc part = HiveFileFormatUtils
.getPartitionDescFromPathRecursively(pathToPartitionInfo, dir,
- IOPrepareCache.get().allocatePartitionDescMap());
+ IOPrepareCache.get().allocatePartitionDescMap(), true);
// create a new InputFormat instance if this is the first time to see this
// class
Class inputFormatClass = part.getInputFileFormatClass();
Index: ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java (revision 1056008)
+++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/HiveCompactIndexResult.java (working copy)
@@ -31,6 +31,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
@@ -74,6 +75,7 @@
JobConf job = null;
BytesRefWritable[] bytesRef = new BytesRefWritable[2];
+ boolean ignoreHdfsLoc = false;
public HiveCompactIndexResult(String indexFile, JobConf conf) throws IOException,
HiveException {
@@ -81,6 +83,7 @@
bytesRef[0] = new BytesRefWritable();
bytesRef[1] = new BytesRefWritable();
+ ignoreHdfsLoc = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_INDEX_IGNORE_HDFS_LOC);
if (indexFile != null) {
Path indexFilePath = new Path(indexFile);
@@ -128,6 +131,11 @@
+ line.toString());
}
String bucketFileName = new String(bytes, 0, firstEnd);
+
+ if (ignoreHdfsLoc) {
+ Path tmpPath = new Path(bucketFileName);
+ bucketFileName = tmpPath.toUri().getPath();
+ }
IBucket bucket = buckets.get(bucketFileName);
if (bucket == null) {
bucket = new IBucket(bucketFileName);
Index: ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (revision 1056008)
+++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (working copy)
@@ -249,11 +249,19 @@
Map pathToPartitionInfo, Path dir,
Map