Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (revision 1135335) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (working copy) @@ -31,7 +31,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Enumeration; +import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; @@ -809,8 +811,8 @@ throws Exception { int numEmptyPaths = 0; - List pathsProcessed = new ArrayList(); - + Set pathsProcessed = new HashSet(); + List pathsToAdd = new LinkedList(); // AliasToWork contains all the aliases for (String oneAlias : work.getAliasToWork().keySet()) { LOG.info("Processing alias " + oneAlias); @@ -828,15 +830,14 @@ if (pathsProcessed.contains(path)) { continue; } + pathsProcessed.add(path); LOG.info("Adding input file " + path); - - Path dirPath = new Path(path); - if (!Utilities.isEmptyPath(job, path, ctx)) { - FileInputFormat.addInputPath(job, dirPath); - } else { + if (Utilities.isEmptyPath(job, path, ctx)) { emptyPaths.add(path); + } else { + pathsToAdd.add(path); } } } @@ -860,6 +861,21 @@ oneAlias); } } + setInputPaths(job, pathsToAdd); + } + + private static void setInputPaths(JobConf job, List pathsToAdd) { + Path[] addedPaths = FileInputFormat.getInputPaths(job); + List toAddPathList = new ArrayList(); + if(addedPaths != null) { + for(Path added: addedPaths) { + toAddPathList.add(added); + } + } + for(String toAdd: pathsToAdd) { + toAddPathList.add(new Path(toAdd)); + } + FileInputFormat.setInputPaths(job, toAddPathList.toArray(new Path[0])); } @Override Index: ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (revision 1135335) +++ ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (working copy) @@ -510,7 +510,7 @@ } static class CombineFilter implements PathFilter { - private final List pStrings = new ArrayList(); + private final Set pStrings = new HashSet(); // store a path prefix in this TestFilter // PRECONDITION: p should always be a directory @@ -522,19 +522,22 @@ } public void addPath(Path p) { - String pString = p.toUri().getPath().toString() + File.separator;; + String pString = p.toUri().getPath().toString(); pStrings.add(pString); } // returns true if the specified path matches the prefix stored // in this TestFilter. public boolean accept(Path path) { - for (String pString : pStrings) { - if (path.toString().indexOf(pString) == 0) { - return true; + boolean find = false; + while (path != null && !find) { + if(pStrings.contains(path.toString())) { + find = true; + break; } + path = path.getParent(); } - return false; + return find; } @Override