diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 3bfd539..47475eb 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -146,7 +146,6 @@ */ public static final HiveConf.ConfVars[] dbVars = { HiveConf.ConfVars.HADOOPBIN, - HiveConf.ConfVars.HADOOPJT, HiveConf.ConfVars.METASTOREWAREHOUSE, HiveConf.ConfVars.SCRATCHDIR }; @@ -231,22 +230,23 @@ // a symbolic name to reference in the Hive source code. Properties with non-null // values will override any values set in the underlying Hadoop configuration. HADOOPBIN("hadoop.bin.path", findHadoopBinary()), - HADOOPFS("fs.default.name", null), HIVE_FS_HAR_IMPL("fs.har.impl", "org.apache.hadoop.hive.shims.HiveHarFileSystem"), - HADOOPMAPFILENAME("map.input.file", null), - HADOOPMAPREDINPUTDIR("mapred.input.dir", null), - HADOOPMAPREDINPUTDIRRECURSIVE("mapred.input.dir.recursive", false), - HADOOPJT("mapred.job.tracker", null), - MAPREDMAXSPLITSIZE("mapred.max.split.size", 256000000L), - MAPREDMINSPLITSIZE("mapred.min.split.size", 1L), - MAPREDMINSPLITSIZEPERNODE("mapred.min.split.size.per.rack", 1L), - MAPREDMINSPLITSIZEPERRACK("mapred.min.split.size.per.node", 1L), + HADOOPFS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPFS"), null), + HADOOPMAPFILENAME(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPFILENAME"), null), + HADOOPMAPREDINPUTDIR(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPREDINPUTDIR"), null), + HADOOPMAPREDINPUTDIRRECURSIVE(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPREDINPUTDIRRECURSIVE"), false), + MAPREDMAXSPLITSIZE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), 256000000L), + MAPREDMINSPLITSIZE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), 1L), + MAPREDMINSPLITSIZEPERNODE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERNODE"), 1L), + MAPREDMINSPLITSIZEPERRACK(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERRACK"), 1L), // The number of reduce tasks per job. Hadoop sets this value to 1 by default // By setting this property to -1, Hive will automatically determine the correct // number of reducers. - HADOOPNUMREDUCERS("mapred.reduce.tasks", -1), - HADOOPJOBNAME("mapred.job.name", null), - HADOOPSPECULATIVEEXECREDUCERS("mapred.reduce.tasks.speculative.execution", true), + HADOOPNUMREDUCERS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPNUMREDUCERS"), -1), + HADOOPJOBNAME(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPJOBNAME"), null), + HADOOPSPECULATIVEEXECREDUCERS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPSPECULATIVEEXECREDUCERS"), true), + MAPREDSETUPCLEANUPNEEDED(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false), + MAPREDTASKCLEANUPNEEDED(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false), // Metastore stuff. Be sure to update HiveConf.metaVars when you add // something here! diff --git ql/src/test/results/clientpositive/overridden_confs.q.out ql/src/test/results/clientpositive/overridden_confs.q.out index b621feb..f97d1ad 100644 --- ql/src/test/results/clientpositive/overridden_confs.q.out +++ ql/src/test/results/clientpositive/overridden_confs.q.out @@ -2,7 +2,6 @@ PREHOOK: query: select count(*) from src PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -Key: mapred.job.tracker, Value: local Key: hive.exec.post.hooks, Value: org.apache.hadoop.hive.ql.hooks.VerifyOverriddenConfigsHook Key: hive.config.doesnt.exit, Value: abc 500 diff --git shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java index f03790b..ec1f18e 100644 --- shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -34,6 +34,8 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.HashMap; +import java.util.Map; import javax.security.auth.Subject; import javax.security.auth.login.LoginException; @@ -753,4 +755,22 @@ public WebHCatJTShim getWebHCatShim(Configuration conf, UserGroupInformation ugi public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { return new ProxyFileSystem(fs, uri); } + @Override + public Map getHadoopConfNames() { + Map ret = new HashMap(); + ret.put("HADOOPFS", "fs.default.name"); + ret.put("HADOOPMAPFILENAME", "map.input.file"); + ret.put("HADOOPMAPREDINPUTDIR", "mapred.input.dir"); + ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapred.input.dir.recursive"); + ret.put("MAPREDMAXSPLITSIZE", "mapred.max.split.size"); + ret.put("MAPREDMINSPLITSIZE", "mapred.min.split.size"); + ret.put("MAPREDMINSPLITSIZEPERNODE", "mapred.min.split.size.per.rack"); + ret.put("MAPREDMINSPLITSIZEPERRACK", "mapred.min.split.size.per.node"); + ret.put("HADOOPNUMREDUCERS", "mapred.reduce.tasks"); + ret.put("HADOOPJOBNAME", "mapred.job.name"); + ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapred.reduce.tasks.speculative.execution"); + ret.put("MAPREDSETUPCLEANUPNEEDED", "mapred.committer.job.setup.cleanup.needed"); + ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed"); + return ret; + } } diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index f2f3dab..d0ff7d4 100644 --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -24,6 +24,8 @@ import java.util.Comparator; import java.util.Iterator; import java.net.URI; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; @@ -390,4 +392,22 @@ public void remove() { public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { return new ProxyFileSystem(fs, uri); } + @Override + public Map getHadoopConfNames() { + Map ret = new HashMap(); + ret.put("HADOOPFS", "fs.default.name"); + ret.put("HADOOPMAPFILENAME", "map.input.file"); + ret.put("HADOOPMAPREDINPUTDIR", "mapred.input.dir"); + ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapred.input.dir.recursive"); + ret.put("MAPREDMAXSPLITSIZE", "mapred.max.split.size"); + ret.put("MAPREDMINSPLITSIZE", "mapred.min.split.size"); + ret.put("MAPREDMINSPLITSIZEPERNODE", "mapred.min.split.size.per.rack"); + ret.put("MAPREDMINSPLITSIZEPERRACK", "mapred.min.split.size.per.node"); + ret.put("HADOOPNUMREDUCERS", "mapred.reduce.tasks"); + ret.put("HADOOPJOBNAME", "mapred.job.name"); + ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapred.reduce.tasks.speculative.execution"); + ret.put("MAPREDSETUPCLEANUPNEEDED", "mapred.committer.job.setup.cleanup.needed"); + ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed"); + return ret; + } } diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 47591d2..2b5fc7e 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -25,6 +25,7 @@ import java.util.Comparator; import java.util.Iterator; import java.util.Map; +import java.util.HashMap; import java.net.URI; import java.io.FileNotFoundException; @@ -456,4 +457,23 @@ public LocatedFileStatus next() throws IOException { public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { return new ProxyFileSystem23(fs, uri); } + + @Override + public Map getHadoopConfNames() { + Map ret = new HashMap(); + ret.put("HADOOPFS", "fs.defaultFS"); + ret.put("HADOOPMAPFILENAME", "mapreduce.map.input.file"); + ret.put("HADOOPMAPREDINPUTDIR", "mapreduce.input.fileinputformat.inputdir"); + ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapreduce.input.fileinputformat.input.dir.recursive"); + ret.put("MAPREDMAXSPLITSIZE", "mapreduce.input.fileinputformat.split.maxsize"); + ret.put("MAPREDMINSPLITSIZE", "mapreduce.input.fileinputformat.split.minsize"); + ret.put("MAPREDMINSPLITSIZEPERNODE", "mapreduce.input.fileinputformat.split.minsize.per.rack"); + ret.put("MAPREDMINSPLITSIZEPERRACK", "mapreduce.input.fileinputformat.split.minsize.per.node"); + ret.put("HADOOPNUMREDUCERS", "mapreduce.job.reduces"); + ret.put("HADOOPJOBNAME", "mapreduce.job.name"); + ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapreduce.reduce.speculative"); + ret.put("MAPREDSETUPCLEANUPNEEDED", "mapreduce.job.committer.setup.cleanup.needed"); + ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed"); + return ret; + } } diff --git shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index 84f3ddc..e205caa 100644 --- shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier; import org.apache.hadoop.hive.thrift.DelegationTokenSelector; import org.apache.hadoop.http.HtmlQuoting; @@ -324,18 +325,18 @@ public void createPool(JobConf conf, PathFilter... filters) { @Override public InputSplitShim[] getSplits(JobConf job, int numSplits) throws IOException { - long minSize = job.getLong("mapred.min.split.size", 0); + long minSize = job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), 0); // For backward compatibility, let the above parameter be used - if (job.getLong("mapred.min.split.size.per.node", 0) == 0) { + if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERNODE"), 0) == 0) { super.setMinSplitSizeNode(minSize); } - if (job.getLong("mapred.min.split.size.per.rack", 0) == 0) { + if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERRACK"), 0) == 0) { super.setMinSplitSizeRack(minSize); } - if (job.getLong("mapred.max.split.size", 0) == 0) { + if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), 0) == 0) { super.setMaxSplitSize(minSize); } @@ -426,11 +427,11 @@ public void prepareJobOutput(JobConf conf) { // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 - conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false); + conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false); // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 - conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false); + conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false); } @Override diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 0e982ee..2b3c6c1 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -28,6 +28,7 @@ import java.util.Comparator; import java.util.Iterator; import java.util.List; +import java.util.Map; import javax.security.auth.login.LoginException; @@ -517,4 +518,6 @@ RecordReader getRecordReader(JobConf job, InputSplitShim split, Reporter reporte * other file system. */ public FileSystem createProxyFileSystem(FileSystem fs, URI uri); + + public Map getHadoopConfNames(); }