diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index 89fff81..c00ff59 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; import org.apache.hadoop.hive.ql.exec.FooterBuffer; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveRecordReader; @@ -748,7 +749,8 @@ public ObjectInspector getOutputObjectInspector() throws HiveException { */ private FileStatus[] listStatusUnderPath(FileSystem fs, Path p) throws IOException { boolean recursive = HiveConf.getBoolVar(job, HiveConf.ConfVars.HADOOPMAPREDINPUTDIRRECURSIVE); - if (!recursive) { + // If this is in acid format always read it recursively regardless of what the jobconf says. + if (!recursive && !AcidUtils.isAcid(p, job)) { return fs.listStatus(p); } List results = new ArrayList(); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index 80ef611..e7c0b38 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -305,6 +305,28 @@ static ParsedDelta parseDelta(FileStatus path) { } /** + * Is the given directory in ACID format? + * @param directory the partition directory to check + * @param conf the query configuration + * @return true, if it is an ACID directory + * @throws IOException + */ + public static boolean isAcid(Path directory, + Configuration conf) throws IOException { + FileSystem fs = directory.getFileSystem(conf); + for(FileStatus file: fs.listStatus(directory)) { + String filename = file.getPath().getName(); + if (filename.startsWith(BASE_PREFIX) || + filename.startsWith(DELTA_PREFIX)) { + if (file.isDirectory()) { + return true; + } + } + } + return false; + } + + /** * Get the ACID state of the given directory. It finds the minimal set of * base and diff directories. Note that because major compactions don't * preserve the history, we can't use a base directory that includes a