diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index dfcd2bc..bf16a9f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -2877,7 +2877,7 @@ public static double getHighestSamplePercentage (MapWork work) { pathsProcessed.add(path); LOG.info("Adding input file " + path); - if (isEmptyPath(job, path, ctx)) { + if (!HiveConf.getBoolVar(job, ConfVars.HIVE_OPTIMIZE_TEZ) && isEmptyPath(job, path, ctx)) { path = createDummyFileForEmptyPartition(path, job, work, hiveScratchDir, alias, sequenceNumber++); @@ -2894,7 +2894,7 @@ public static double getHighestSamplePercentage (MapWork work) { // T2) x; // If T is empty and T2 contains 100 rows, the user expects: 0, 100 (2 // rows) - if (path == null) { + if (path == null && !HiveConf.getBoolVar(job, ConfVars.HIVE_OPTIMIZE_TEZ)) { path = createDummyFileForEmptyTable(job, work, hiveScratchDir, alias, sequenceNumber++); pathsToAdd.add(path); @@ -3174,5 +3174,9 @@ private static long getSize(HiveConf conf, Partition partition) { return getSize(conf, size, path); } + + public static void clearWorkMap() { + gWorkMap.clear(); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 1726da6..661be21 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -522,13 +522,7 @@ public static LocalResource localizeResource(Path src, Path dest, Configuration public static JobConf createConfiguration(HiveConf hiveConf) throws IOException { hiveConf.setBoolean("mapred.mapper.new-api", false); - JobConf conf = (JobConf) MRHelpers.getBaseMRConfiguration(); - - for (Map.Entry entry: hiveConf) { - if (conf.get(entry.getKey()) == null) { - conf.set(entry.getKey(), entry.getValue()); - } - } + JobConf conf = (JobConf) MRHelpers.getBaseMRConfiguration(hiveConf); conf.set("mapred.output.committer.class", NullOutputCommitter.class.getName()); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java index ca6f4e4..5e21195 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java @@ -234,6 +234,7 @@ void close(){ throw new RuntimeException("Hive Runtime Error while closing operators", e); } } finally { + Utilities.clearWorkMap(); MapredContext.close(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java index 5db7e52..3958671 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java @@ -357,6 +357,7 @@ void close(){ + e.getMessage(), e); } } finally { + Utilities.clearWorkMap(); MapredContext.close(); } }