commit d2338f9cf6ab7d168ad62e01896d979e21e4d43f Author: Mithun RK Date: Wed Sep 27 11:10:36 2017 -0700 HIVE-17621: Hive-site settings are ignored during HCatInputFormat split-calculation (Chris Drome, via Mithun Radhakrishnan) diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java index 8b927af8a2..d18e2068bc 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java @@ -33,6 +33,8 @@ import java.util.Map; import java.util.Properties; +import javax.security.auth.login.LoginException; + import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -76,8 +78,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.security.auth.login.LoginException; - public class HCatUtil { private static final Logger LOG = LoggerFactory.getLogger(HCatUtil.class); @@ -642,6 +642,25 @@ public static JobConf getJobConfFromContext(JobContext jobContext) { return jobConf; } + public static Map getHCatKeyHiveConf(JobConf conf) { + Map hiveProperties = new HashMap(); + if (conf.get(HCatConstants.HCAT_KEY_HIVE_CONF) != null) { + try { + Properties properties = (Properties) HCatUtil.deserialize( + conf.get(HCatConstants.HCAT_KEY_HIVE_CONF)); + for (Map.Entry prop : properties.entrySet()) { + if (conf.get((String)prop.getKey()) == null) { + hiveProperties.put(prop.getKey().toString(), prop.getValue().toString()); + } + } + } catch (IOException e) { + throw new IllegalStateException( + "Failed to deserialize hive conf", e); + } + } + return hiveProperties; + } + public static void copyJobPropertiesToJobConf( Map jobProperties, JobConf jobConf) { for (Map.Entry entry : jobProperties.entrySet()) { @@ -649,7 +668,6 @@ public static void copyJobPropertiesToJobConf( } } - public static boolean isHadoop23() { String version = org.apache.hadoop.util.VersionInfo.getVersion(); if (version.matches("\\b0\\.23\\..+\\b")||version.matches("\\b2\\..*")) diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java index 9caff7a97d..ec1e1ca1bb 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java @@ -126,16 +126,20 @@ public static void setOutputSchema(Job job, HCatSchema hcatSchema) } HiveStorageHandler storageHandler; - JobConf jobConf; + Map hiveProps = null; //For each matching partition, call getSplits on the underlying InputFormat for (PartInfo partitionInfo : partitionInfoList) { - jobConf = HCatUtil.getJobConfFromContext(jobContext); + JobConf jobConf = HCatUtil.getJobConfFromContext(jobContext); + if (hiveProps == null) { + hiveProps = HCatUtil.getHCatKeyHiveConf(jobConf); + } List setInputPath = setInputPath(jobConf, partitionInfo.getLocation()); if (setInputPath.isEmpty()) { continue; } Map jobProperties = partitionInfo.getJobProperties(); + HCatUtil.copyJobPropertiesToJobConf(hiveProps, jobConf); HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf); storageHandler = HCatUtil.getStorageHandler(