diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 33b67dd7b0fde41f81f8d86ea8c83d29c631e3d7..e7ec63410df55dca4e6b85c24a7eadaba9039f32 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1175,8 +1175,6 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVEROWOFFSET("hive.exec.rowoffset", false, "Whether to provide the row offset virtual column"), - HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE("hive.hadoop.supports.splittable.combineinputformat", false, ""), - // Optimizer HIVEOPTINDEXFILTER("hive.optimize.index.filter", false, "Whether to enable automatic use of indexes"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java index 1de7e4073f5eea4c7be8423a7ebe6a89cb51d9f1..bbbbf2b0839d94eb444c0035f2d2bd09d3f1b4f6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java @@ -373,45 +373,6 @@ public int hashCode() { } FileSystem inpFs = path.getFileSystem(job); - // Since there is no easy way of knowing whether MAPREDUCE-1597 is present in the tree or not, - // we use a configuration variable for the same - if (this.mrwork != null && !this.mrwork.getHadoopSupportsSplittable()) { - // The following code should be removed, once - // https://issues.apache.org/jira/browse/MAPREDUCE-1597 is fixed. - // Hadoop does not handle non-splittable files correctly for CombineFileInputFormat, - // so don't use CombineFileInputFormat for non-splittable files - - //ie, dont't combine if inputformat is a TextInputFormat and has compression turned on - - if (inputFormat instanceof TextInputFormat) { - Queue dirs = new LinkedList(); - FileStatus fStats = inpFs.getFileStatus(path); - - // If path is a directory - if (fStats.isDir()) { - dirs.offer(path); - } else if ((new CompressionCodecFactory(job)).getCodec(path) != null) { - //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine) - splits = super.getSplits(job, numSplits); - return splits; - } - - while (dirs.peek() != null) { - Path tstPath = dirs.remove(); - FileStatus[] fStatus = inpFs.listStatus(tstPath, FileUtils.HIDDEN_FILES_PATH_FILTER); - for (int idx = 0; idx < fStatus.length; idx++) { - if (fStatus[idx].isDir()) { - dirs.offer(fStatus[idx].getPath()); - } else if ((new CompressionCodecFactory(job)).getCodec( - fStatus[idx].getPath()) != null) { - //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine) - splits = super.getSplits(job, numSplits); - return splits; - } - } - } - } - } //don't combine if inputformat is a SymlinkTextInputFormat if (inputFormat instanceof SymlinkTextInputFormat) { splits = super.getSplits(job, numSplits); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 693d8c7e9f956999b2da33593d780e37ddf2b3b8..4a325fbb8c516d4144ca0d24602b74ccb559d438 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -933,8 +933,6 @@ public static MapredWork getMapRedWorkFromConf(HiveConf conf) { work.setPathToAliases(new LinkedHashMap>()); work.setPathToPartitionInfo(new LinkedHashMap()); work.setAliasToWork(new LinkedHashMap>()); - work.setHadoopSupportsSplittable( - conf.getBoolVar(HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE)); return mrWork; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index 3217df27bb5731a1dcd5db1ae17c5bdff2e3fbfc..8eb99fde8d4d08859556a5b17f45200129bbdb51 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -63,8 +63,6 @@ private static final Log LOG = LogFactory.getLog(MapWork.class); - private boolean hadoopSupportsSplittable; - // use LinkedHashMap to make sure the iteration order is // deterministic, to ease testing private LinkedHashMap> pathToAliases = new LinkedHashMap>(); @@ -419,14 +417,6 @@ public boolean isMapperCannotSpanPartns() { return this.mapperCannotSpanPartns; } - public boolean getHadoopSupportsSplittable() { - return hadoopSupportsSplittable; - } - - public void setHadoopSupportsSplittable(boolean hadoopSupportsSplittable) { - this.hadoopSupportsSplittable = hadoopSupportsSplittable; - } - public String getIndexIntermediateFile() { return indexIntermediateFile; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 76926e79729d9ea4823de0ffc9b1e5bac6364842..b50eaabdbed8f1270a4cf8aef4c3a846fdae3745 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -99,14 +99,7 @@ public static synchronized long getCountForMapJoinDumpFilePrefix() { @SuppressWarnings("nls") public static MapredWork getMapRedWork() { - try { - MapredWork work = new MapredWork(); - work.getMapWork().setHadoopSupportsSplittable(Hive.get().getConf().getBoolVar( - HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE)); - return work; - } catch (HiveException ex) { - throw new RuntimeException(ex); - } + return new MapredWork(); } public static TableDesc getDefaultTableDesc(CreateTableDesc directoryDesc,