diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index be85242..ee42f4c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -388,7 +388,6 @@ private static BaseWork getBaseWork(Configuration conf, String name) { in = new InflaterInputStream(in); } else { LOG.info("Open file to read in plan: " + localPath); -// in = new FileInputStream(localPath.toUri().getPath()); in = localPath.getFileSystem(conf).open(localPath); } @@ -427,8 +426,9 @@ private static BaseWork getBaseWork(Configuration conf, String name) { LOG.info("No plan file found: "+path); return null; } catch (Exception e) { - LOG.error("Failed to load plan: "+path, e); - throw new RuntimeException(e); + String msg = "Failed to load plan: " + path + ": " + e; + LOG.error(msg, e); + throw new RuntimeException(msg, e); } finally { if (in != null) { try { @@ -710,11 +710,11 @@ private static Path setBaseWork(Configuration conf, BaseWork w, Path hiveScratch // Cache the plan in this process gWorkMap.put(planPath, w); - return planPath; } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException(e); + String msg = "Error caching " + name + ": " + e; + LOG.error(msg, e); + throw new RuntimeException(msg, e); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java index 7757367..b9d017e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java @@ -82,8 +82,9 @@ */ public static class CombineHiveInputSplit extends InputSplitShim { - String inputFormatClassName; - CombineFileSplit inputSplitShim; + private String inputFormatClassName; + private CombineFileSplit inputSplitShim; + private Map pathToPartitionInfo; public CombineHiveInputSplit() throws IOException { this(ShimLoader.getHadoopShims().getCombineFileInputFormat() @@ -93,20 +94,25 @@ public CombineHiveInputSplit() throws IOException { public CombineHiveInputSplit(CombineFileSplit inputSplitShim) throws IOException { this(inputSplitShim.getJob(), inputSplitShim); } - public CombineHiveInputSplit(JobConf job, CombineFileSplit inputSplitShim) throws IOException { + this(job, inputSplitShim, null); + } + public CombineHiveInputSplit(JobConf job, CombineFileSplit inputSplitShim, + Map pathToPartitionInfo) throws IOException { this.inputSplitShim = inputSplitShim; + this.pathToPartitionInfo = pathToPartitionInfo; if (job != null) { - Map pathToPartitionInfo = Utilities - .getMapWork(job).getPathToPartitionInfo(); + if (this.pathToPartitionInfo == null) { + this.pathToPartitionInfo = Utilities.getMapWork(job).getPathToPartitionInfo(); + } // extract all the inputFormatClass names for each chunk in the // CombinedSplit. Path[] ipaths = inputSplitShim.getPaths(); if (ipaths.length > 0) { PartitionDesc part = HiveFileFormatUtils - .getPartitionDescFromPathRecursively(pathToPartitionInfo, + .getPartitionDescFromPathRecursively(this.pathToPartitionInfo, ipaths[0], IOPrepareCache.get().getPartitionDescMap()); inputFormatClassName = part.getInputFileFormatClass().getName(); } @@ -215,8 +221,9 @@ public void write(DataOutput out) throws IOException { inputSplitShim.write(out); if (inputFormatClassName == null) { - Map pathToPartitionInfo = Utilities - .getMapWork(getJob()).getPathToPartitionInfo(); + if (pathToPartitionInfo == null) { + pathToPartitionInfo = Utilities.getMapWork(getJob()).getPathToPartitionInfo(); + } // extract all the inputFormatClass names for each chunk in the // CombinedSplit. @@ -268,8 +275,8 @@ public int hashCode() { /** * Create Hive splits based on CombineFileSplit. */ - private InputSplit[] getCombineSplits(JobConf job, - int numSplits) throws IOException { + private InputSplit[] getCombineSplits(JobConf job, int numSplits, Map pathToPartitionInfo) + throws IOException { PerfLogger perfLogger = PerfLogger.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.GET_SPLITS); init(job); @@ -438,7 +445,7 @@ public int hashCode() { } for (CombineFileSplit is : iss) { - CombineHiveInputSplit csplit = new CombineHiveInputSplit(job, is); + CombineHiveInputSplit csplit = new CombineHiveInputSplit(job, is, pathToPartitionInfo); result.add(csplit); } @@ -505,7 +512,8 @@ public int hashCode() { if (combinablePaths.size() > 0) { FileInputFormat.setInputPaths(job, combinablePaths.toArray (new Path[combinablePaths.size()])); - InputSplit[] splits = getCombineSplits(job, numSplits); + Map pathToPartitionInfo = Utilities.getMapWork(job).getPathToPartitionInfo(); + InputSplit[] splits = getCombineSplits(job, numSplits, pathToPartitionInfo); for (InputSplit split : splits) { result.add(split); }