diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index a617a1af46..e7b02ddf23 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4928,6 +4928,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Merge adjacent joins into a single n-way join"), HIVE_LOG_N_RECORDS("hive.log.every.n.records", 0L, new RangeValidator(0L, null), "If value is greater than 0 logs in fixed intervals of size n rather than exponentially."), + HIVE_TEZ_SKIP_LOCAL_XML("hive.tez.skip.local.xml", true, "Hive excludes local xml files" + + " when sending configuration options to Tez AM when true"), /** * @deprecated Use MetastoreConf.MSCK_PATH_VALIDATION */ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index d21aa45d79..1fedad9e85 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -1371,7 +1371,8 @@ public JobConf createConfiguration(HiveConf hiveConf, boolean skipAMConf) throws hiveConf.setBoolean("mapred.mapper.new-api", false); Predicate findDefaults = - (s) -> ((s != null) && (s.endsWith(".xml") || (s.endsWith(".java") && !"HiveConf.java".equals(s)))); + (s) -> ((s != null) && ((s.endsWith(".xml") && !s.endsWith("hive-site.xml")) || + (s.endsWith(".java") && !"HiveConf.java".equals(s)))); // since this is an inclusion filter, negate the predicate JobConf conf = diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java index 67d6982ac0..d50a36ce6c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java @@ -127,7 +127,8 @@ public HiveSplitGenerator(InputInitializerContext initializerContext) throws IOE userPayloadProto = MRInputHelpers.parseMRInputPayload(initializerContext.getInputUserPayload()); - this.conf = TezUtils.createConfFromByteString(userPayloadProto.getConfigurationBytes()); + this.conf = new Configuration(initializerContext.getVertexConfiguration()); + TezUtils.addToConfFromByteString(this.conf, userPayloadProto.getConfigurationBytes()); this.jobConf = new JobConf(conf); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java index fa6160fe3c..590de42735 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java @@ -173,7 +173,7 @@ public void handleEvents(List arg0) { @Override public void initialize() throws IOException { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INITIALIZE_PROCESSOR); - Configuration conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); + Configuration conf = TezUtils.createConfFromBaseConfAndPayload(getContext()); this.jobConf = new JobConf(conf); this.processorContext = getContext(); ExecutionContext execCtx = processorContext.getExecutionContext(); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 3d276323f5..902242dc49 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -176,8 +176,9 @@ public int execute() { ss.getHiveVariables().get("wmpool"), ss.getHiveVariables().get("wmapp")); WmContext wmContext = ctx.getWmContext(); + boolean skipXmlConfs = conf.getBoolVar(HiveConf.ConfVars.HIVE_TEZ_SKIP_LOCAL_XML); // jobConf will hold all the configuration for hadoop, tez, and hive, which are not set in AM defaults - JobConf jobConf = utils.createConfiguration(conf, false); + JobConf jobConf = utils.createConfiguration(conf, skipXmlConfs); // Get all user jars from work (e.g. input format stuff).