diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java index 8e48c2e..c822d7b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java @@ -133,8 +133,8 @@ public HiveSplitGenerator(InputInitializerContext initializerContext) throws IOE (InputFormat) ReflectionUtils.newInstance(JavaUtils.loadClass(realInputFormatName), jobConf); - int totalResource = getContext().getTotalAvailableResource().getMemory(); - int taskResource = getContext().getVertexTaskResource().getMemory(); + int totalResource = Math.max(getContext().getTotalAvailableResource().getMemory(), 0); + int taskResource = Math.max(getContext().getVertexTaskResource().getMemory(), 1); int availableSlots = totalResource / taskResource; if (HiveConf.getLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1) <= 1) { @@ -154,8 +154,12 @@ public HiveSplitGenerator(InputInitializerContext initializerContext) throws IOE conf.getFloat(TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES, TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES_DEFAULT); + int numSplits = (int) (availableSlots * waves); + if (numSplits <= 0) { + numSplits = 1; + } // Raw splits - InputSplit[] splits = inputFormat.getSplits(jobConf, (int) (availableSlots * waves)); + InputSplit[] splits = inputFormat.getSplits(jobConf, numSplits); // Sort the splits, so that subsequent grouping is consistent. Arrays.sort(splits, new InputSplitComparator()); LOG.info("Number of input splits: " + splits.length + ". " + availableSlots