diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 3e8ba08d11..c9aab27472 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -315,8 +315,8 @@ private void getCredentialsForFileSinks(BaseWork baseWork, DAG dag) { * Creates the configuration object necessary to run a specific vertex from * map work. This includes input formats, input processor, etc. */ - private JobConf initializeVertexConf(JobConf baseConf, Context context, MapWork mapWork) { - JobConf conf = new JobConf(baseConf); + private XmlFilteringJobConf initializeVertexConf(JobConf baseConf, Context context, MapWork mapWork) { + XmlFilteringJobConf conf = new XmlFilteringJobConf(baseConf); conf.set(Operator.CONTEXT_NAME_KEY, mapWork.getName()); @@ -862,8 +862,8 @@ private Vertex createVertexFromMapWork(JobConf conf, MapWork mapWork, Path mrScr /* * Helper function to create JobConf for specific ReduceWork. */ - private JobConf initializeVertexConf(JobConf baseConf, Context context, ReduceWork reduceWork) { - JobConf conf = new JobConf(baseConf); + private XmlFilteringJobConf initializeVertexConf(JobConf baseConf, Context context, ReduceWork reduceWork) { + XmlFilteringJobConf conf = new XmlFilteringJobConf(baseConf); conf.set(Operator.CONTEXT_NAME_KEY, reduceWork.getName()); @@ -1409,7 +1409,7 @@ public JobConf createConfiguration(HiveConf hiveConf, boolean skipAMConf) throws * @param work BaseWork will be used to populate the configuration object. * @return JobConf new configuration object */ - public JobConf initializeVertexConf(JobConf conf, Context context, BaseWork work) { + public XmlFilteringJobConf initializeVertexConf(JobConf conf, Context context, BaseWork work) { // simply dispatch the call to the right method for the actual (sub-) type of // BaseWork. @@ -1425,7 +1425,7 @@ public JobConf initializeVertexConf(JobConf conf, Context context, BaseWork work } } - private JobConf initializeVertexConf(JobConf conf, Context context, MergeJoinWork work) { + private XmlFilteringJobConf initializeVertexConf(JobConf conf, Context context, MergeJoinWork work) { if (work.getMainWork() instanceof MapWork) { return initializeVertexConf(conf, context, (MapWork) (work.getMainWork())); } else { @@ -1445,7 +1445,6 @@ private JobConf initializeVertexConf(JobConf conf, Context context, MergeJoinWor @SuppressWarnings("deprecation") public Vertex createVertex(JobConf conf, BaseWork workUnit, Path scratchDir, TezWork tezWork, Map localResources) throws Exception { - Vertex vertex; // simply dispatch the call to the right method for the actual (sub-) type of // BaseWork. diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java index 84ae54157e..fa8e63b9e2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezConfigurationFactory.java @@ -78,7 +78,7 @@ public static Configuration copyInto(Configuration target, Configuration src, } if (sourceFilter == null || sourceFilter.test(source)) { - target.set(name, value); + target.set(name, value, source); } else { } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/XmlFilteringJobConf.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/XmlFilteringJobConf.java new file mode 100644 index 0000000000..ad0ae2e493 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/XmlFilteringJobConf.java @@ -0,0 +1,41 @@ +package org.apache.hadoop.hive.ql.exec.tez; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * Iterator of this class filters out configurations set in .xml files. + */ +public class XmlFilteringJobConf extends JobConf { + + public XmlFilteringJobConf(Configuration other) { + super(other); + } + + @Override + public Iterator> iterator() { + Map result = new HashMap(); + for(Map.Entry item: getProps().entrySet()) { + if (item.getKey() instanceof String && + item.getValue() instanceof String) { + String name = (String) item.getKey(); + if (!isFromXml(name)) { + result.put((String) item.getKey(), (String) item.getValue()); + } + } + } + return result.entrySet().iterator(); + } + + private boolean isFromXml(String name) { + String[] propSources = getPropertySources(name); + if (propSources != null && propSources[propSources.length - 1].endsWith(".xml") ) { + return true; + } + return false; + } +}