diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 01cd731..dad84f1 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hive.conf.Validator.RatioValidator; import org.apache.hadoop.hive.conf.Validator.StringSet; import org.apache.hadoop.hive.conf.Validator.TimeValidator; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.UserGroupInformation; @@ -386,22 +385,22 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HADOOPBIN("hadoop.bin.path", findHadoopBinary(), "", true), HIVE_FS_HAR_IMPL("fs.har.impl", "org.apache.hadoop.hive.shims.HiveHarFileSystem", "The implementation for accessing Hadoop Archives. Note that this won't be applicable to Hadoop versions less than 0.20"), - HADOOPFS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPFS"), null, "", true), - HADOOPMAPFILENAME(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPFILENAME"), null, "", true), - HADOOPMAPREDINPUTDIR(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPREDINPUTDIR"), null, "", true), - HADOOPMAPREDINPUTDIRRECURSIVE(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPREDINPUTDIRRECURSIVE"), false, "", true), - MAPREDMAXSPLITSIZE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), 256000000L, "", true), - MAPREDMINSPLITSIZE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), 1L, "", true), - MAPREDMINSPLITSIZEPERNODE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERNODE"), 1L, "", true), - MAPREDMINSPLITSIZEPERRACK(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERRACK"), 1L, "", true), + HADOOPFS("fs.defaultFS", null, "", true), + HADOOPMAPFILENAME("mapreduce.map.input.file", null, "", true), + HADOOPMAPREDINPUTDIR("mapreduce.input.fileinputformat.inputdir", null, "", true), + HADOOPMAPREDINPUTDIRRECURSIVE("mapreduce.input.fileinputformat.input.dir.recursive", false, "", true), + MAPREDMAXSPLITSIZE("mapreduce.input.fileinputformat.split.maxsize", 256000000L, "", true), + MAPREDMINSPLITSIZE("mapreduce.input.fileinputformat.split.minsize", 1L, "", true), + MAPREDMINSPLITSIZEPERNODE("mapreduce.input.fileinputformat.split.minsize.per.node", 1L, "", true), + MAPREDMINSPLITSIZEPERRACK("mapreduce.input.fileinputformat.split.minsize.per.rack", 1L, "", true), // The number of reduce tasks per job. Hadoop sets this value to 1 by default // By setting this property to -1, Hive will automatically determine the correct // number of reducers. - HADOOPNUMREDUCERS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPNUMREDUCERS"), -1, "", true), - HADOOPJOBNAME(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPJOBNAME"), null, "", true), - HADOOPSPECULATIVEEXECREDUCERS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPSPECULATIVEEXECREDUCERS"), true, "", true), - MAPREDSETUPCLEANUPNEEDED(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false, "", true), - MAPREDTASKCLEANUPNEEDED(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false, "", true), + HADOOPNUMREDUCERS("mapreduce.job.reduces", -1, "", true), + HADOOPJOBNAME("mapreduce.job.name", null, "", true), + HADOOPSPECULATIVEEXECREDUCERS("mapreduce.reduce.speculative", true, "", true), + MAPREDSETUPCLEANUPNEEDED("mapreduce.job.committer.setup.cleanup.needed", false, "", true), + MAPREDTASKCLEANUPNEEDED("mapreduce.job.committer.task.cleanup.needed", false, "", true), // Metastore stuff. Be sure to update HiveConf.metaVars when you add something here! METASTOREWAREHOUSE("hive.metastore.warehouse.dir", "/user/hive/warehouse", diff --git common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java index a3aa549..0a28ca2 100644 --- common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java +++ common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java @@ -22,11 +22,9 @@ import com.codahale.metrics.Timer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.common.metrics.common.MetricsVariable; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.shims.ShimLoader; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -57,8 +55,7 @@ public void before() throws Exception { jsonReportFile = new File(workDir, "json_reporting"); jsonReportFile.delete(); - String defaultFsName = ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPFS"); - conf.set(defaultFsName, "local"); + HiveConf.setVar(conf, HiveConf.ConfVars.HADOOPFS, "local"); conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name()); conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); diff --git hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java index 0c3d707..d5fa93c 100644 --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java @@ -21,8 +21,8 @@ import java.io.IOException; import java.util.List; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.mapreduce.InputSplit; @@ -44,10 +44,8 @@ @Override public List getSplits(JobContext job) throws IOException { - - job.getConfiguration().setLong( - ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), - SequenceFile.SYNC_INTERVAL); + HiveConf.setLongVar(job.getConfiguration(), + HiveConf.ConfVars.MAPREDMINSPLITSIZE, SequenceFile.SYNC_INTERVAL); return super.getSplits(job); } } diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java index 02a13b3..bb6c582 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java @@ -230,8 +230,8 @@ private void writeThenReadByRecordReader(int intervalRecordCount, Configuration jonconf = new Configuration(cloneConf); jonconf.set("mapred.input.dir", testDir.toString()); JobContext context = new Job(jonconf); - context.getConfiguration().setLong( - ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), maxSplitSize); + HiveConf.setLongVar(context.getConfiguration(), + HiveConf.ConfVars.MAPREDMAXSPLITSIZE, maxSplitSize); List splits = inputFormat.getSplits(context); assertEquals("splits length should be " + splitNumber, splits.size(), splitNumber); int readCount = 0; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java index c370381..f902666 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.FileSplit; -import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; @@ -81,16 +80,6 @@ private final MapWork work; private final SplitGrouper splitGrouper = new SplitGrouper(); - private static final String MIN_SPLIT_SIZE; - @SuppressWarnings("unused") - private static final String MAX_SPLIT_SIZE; - - static { - final HadoopShims SHIMS = ShimLoader.getHadoopShims(); - MIN_SPLIT_SIZE = SHIMS.getHadoopConfNames().get("MAPREDMINSPLITSIZE"); - MAX_SPLIT_SIZE = SHIMS.getHadoopConfNames().get("MAPREDMAXSPLITSIZE"); - } - public HiveSplitGenerator(InputInitializerContext initializerContext) throws IOException, SerDeException { super(initializerContext); @@ -143,7 +132,7 @@ public HiveSplitGenerator(InputInitializerContext initializerContext) throws IOE int taskResource = getContext().getVertexTaskResource().getMemory(); int availableSlots = totalResource / taskResource; - if (conf.getLong(MIN_SPLIT_SIZE, 1) <= 1) { + if (HiveConf.getLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1) <= 1) { // broken configuration from mapred-default.xml final long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); @@ -151,7 +140,7 @@ public HiveSplitGenerator(InputInitializerContext initializerContext) throws IOE TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE, TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT); final long preferredSplitSize = Math.min(blockSize / 2, minGrouping); - jobConf.setLong(MIN_SPLIT_SIZE, preferredSplitSize); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, preferredSplitSize); LOG.info("The preferred split size is " + preferredSplitSize); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java index 0328a23..259ed6c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java @@ -51,7 +51,6 @@ import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.compress.CompressionCodec; @@ -583,10 +582,10 @@ public static void prepareJobOutput(JobConf conf) { // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 - conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.MAPREDSETUPCLEANUPNEEDED, false); // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 - conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.MAPREDTASKCLEANUPNEEDED, false); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index a883124..779b960 100755 --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -54,7 +54,6 @@ import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.FileInputFormat; @@ -360,9 +359,7 @@ private void addSplitsForGroup(List dirs, TableScanOperator tableScan, Job footerCount = Utilities.getFooterCount(table, conf); if (headerCount != 0 || footerCount != 0) { // Input file has header or footer, cannot be splitted. - conf.setLong( - ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), - Long.MAX_VALUE); + HiveConf.setLongVar(conf, ConfVars.MAPREDMINSPLITSIZE, Long.MAX_VALUE); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index af40137..7c7b84f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -126,10 +126,6 @@ private static final Logger LOG = LoggerFactory.getLogger(OrcInputFormat.class); private static boolean isDebugEnabled = LOG.isDebugEnabled(); static final HadoopShims SHIMS = ShimLoader.getHadoopShims(); - static final String MIN_SPLIT_SIZE = - SHIMS.getHadoopConfNames().get("MAPREDMINSPLITSIZE"); - static final String MAX_SPLIT_SIZE = - SHIMS.getHadoopConfNames().get("MAPREDMAXSPLITSIZE"); private static final long DEFAULT_MIN_SPLIT_SIZE = 16 * 1024 * 1024; private static final long DEFAULT_MAX_SPLIT_SIZE = 256 * 1024 * 1024; @@ -482,8 +478,8 @@ public boolean validateInput(FileSystem fs, HiveConf conf, Context(Configuration conf, final int minSplits) { this.conf = conf; this.sarg = ConvertAstToSearchArg.createFromConf(conf); - minSize = conf.getLong(MIN_SPLIT_SIZE, DEFAULT_MIN_SPLIT_SIZE); - maxSize = conf.getLong(MAX_SPLIT_SIZE, DEFAULT_MAX_SPLIT_SIZE); + minSize = HiveConf.getLongVar(conf, ConfVars.MAPREDMINSPLITSIZE, DEFAULT_MIN_SPLIT_SIZE); + maxSize = HiveConf.getLongVar(conf, ConfVars.MAPREDMAXSPLITSIZE, DEFAULT_MAX_SPLIT_SIZE); String ss = conf.get(ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname); if (ss == null || ss.equals(SplitStrategyKind.HYBRID.name())) { splitStrategyKind = SplitStrategyKind.HYBRID; diff --git ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java index a68049f..2914194 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java @@ -59,7 +59,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; @@ -642,9 +641,7 @@ public void testSync() throws IOException { RCFileInputFormat inputFormat = new RCFileInputFormat(); JobConf jobconf = new JobConf(cloneConf); jobconf.set("mapred.input.dir", testDir.toString()); - jobconf.setLong( - ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), - fileLen); + HiveConf.setLongVar(jobconf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, fileLen); InputSplit[] splits = inputFormat.getSplits(jobconf, 1); RCFileRecordReader rr = new RCFileRecordReader(jobconf, (FileSplit)splits[0]); long lastSync = 0; @@ -711,9 +708,7 @@ private void writeThenReadByRecordReader(int intervalRecordCount, RCFileInputFormat inputFormat = new RCFileInputFormat(); JobConf jonconf = new JobConf(cloneConf); jonconf.set("mapred.input.dir", testDir.toString()); - jonconf.setLong( - ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), - minSplitSize); + HiveConf.setLongVar(jonconf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, minSplitSize); InputSplit[] splits = inputFormat.getSplits(jonconf, splitNumber); assertEquals("splits length should be " + splitNumber, splits.length, splitNumber); int readCount = 0; diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index c61d615..35e6722 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -994,8 +994,8 @@ public void testSplitGenerator() throws Exception { new MockBlock("host0", "host3-2", "host3-3"), new MockBlock("host4-1", "host4-2", "host4-3"), new MockBlock("host5-1", "host5-2", "host5-3"))); - conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 300); - conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 200); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 300); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 200); OrcInputFormat.Context context = new OrcInputFormat.Context(conf); OrcInputFormat.SplitGenerator splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, @@ -1018,8 +1018,8 @@ public void testSplitGenerator() throws Exception { assertEquals(1800, result.getStart()); assertEquals(200, result.getLength()); // test min = 0, max = 0 generates each stripe - conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 0); - conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 0); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 0); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 0); context = new OrcInputFormat.Context(conf); splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, AcidUtils.createOriginalObj(null, fs.getFileStatus(new Path("/a/file"))), null, true, @@ -1043,8 +1043,8 @@ public void testProjectedColumnSize() throws Exception { new MockBlock("host0", "host3-2", "host3-3"), new MockBlock("host4-1", "host4-2", "host4-3"), new MockBlock("host5-1", "host5-2", "host5-3"))); - conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 300); - conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 200); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 300); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 200); conf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false); conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0"); OrcInputFormat.Context context = new OrcInputFormat.Context(conf); @@ -1068,8 +1068,8 @@ public void testProjectedColumnSize() throws Exception { assertEquals(41867, result.getProjectedColumnsUncompressedSize()); // test min = 0, max = 0 generates each stripe - conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 0); - conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 0); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 0); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 0); context = new OrcInputFormat.Context(conf); splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, fs.getFileStatus(new Path("/a/file")), null, true, @@ -1088,8 +1088,8 @@ public void testProjectedColumnSize() throws Exception { } // single split - conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 100000); - conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 1000); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 1000); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 100000); context = new OrcInputFormat.Context(conf); splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, fs.getFileStatus(new Path("/a/file")), null, true, diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java index 02f015c..3560c43 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -40,7 +41,6 @@ import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.InputFormat; @@ -109,8 +109,8 @@ public void testSplitEliminationSmallMaxSplit() throws Exception { 100000, CompressionKind.NONE, 10000, 10000); writeData(writer); writer.close(); - conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000"); - conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "5000"); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1000); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 5000); InputFormat in = new OrcInputFormat(); FileInputFormat.setInputPaths(conf, testFilePath.toString()); @@ -187,8 +187,8 @@ public void testSplitEliminationLargeMaxSplit() throws Exception { 100000, CompressionKind.NONE, 10000, 10000); writeData(writer); writer.close(); - conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000"); - conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "150000"); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1000); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 150000); InputFormat in = new OrcInputFormat(); FileInputFormat.setInputPaths(conf, testFilePath.toString()); @@ -276,8 +276,8 @@ public void testSplitEliminationComplexExpr() throws Exception { 100000, CompressionKind.NONE, 10000, 10000); writeData(writer); writer.close(); - conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000"); - conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "150000"); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1000); + HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMAXSPLITSIZE, 150000); InputFormat in = new OrcInputFormat(); FileInputFormat.setInputPaths(conf, testFilePath.toString()); diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 4da98e4..0b5c092 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -21,21 +21,16 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URI; -import java.net.URL; import java.nio.ByteBuffer; import java.security.AccessControlException; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -1041,25 +1036,6 @@ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { } @Override - public Map getHadoopConfNames() { - Map ret = new HashMap(); - ret.put("HADOOPFS", "fs.defaultFS"); - ret.put("HADOOPMAPFILENAME", "mapreduce.map.input.file"); - ret.put("HADOOPMAPREDINPUTDIR", "mapreduce.input.fileinputformat.inputdir"); - ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapreduce.input.fileinputformat.input.dir.recursive"); - ret.put("MAPREDMAXSPLITSIZE", "mapreduce.input.fileinputformat.split.maxsize"); - ret.put("MAPREDMINSPLITSIZE", "mapreduce.input.fileinputformat.split.minsize"); - ret.put("MAPREDMINSPLITSIZEPERNODE", "mapreduce.input.fileinputformat.split.minsize.per.node"); - ret.put("MAPREDMINSPLITSIZEPERRACK", "mapreduce.input.fileinputformat.split.minsize.per.rack"); - ret.put("HADOOPNUMREDUCERS", "mapreduce.job.reduces"); - ret.put("HADOOPJOBNAME", "mapreduce.job.name"); - ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapreduce.reduce.speculative"); - ret.put("MAPREDSETUPCLEANUPNEEDED", "mapreduce.job.committer.setup.cleanup.needed"); - ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed"); - return ret; - } - - @Override public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in, ByteBufferPoolShim pool) throws IOException { if(zeroCopy) { return ZeroCopyShims.getZeroCopyReader(in, pool); diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 47b3caa..f9785ec 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -27,7 +27,6 @@ import java.security.NoSuchAlgorithmException; import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -411,8 +410,6 @@ public void setFullFileStatus(Configuration conf, HdfsFileStatus sourceStatus, */ public FileSystem createProxyFileSystem(FileSystem fs, URI uri); - public Map getHadoopConfNames(); - /** * Create a shim for DFS storage policy. */ diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index 0a0f52d..ea66ee2 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -294,18 +294,18 @@ public void createPool(JobConf conf, PathFilter... filters) { @Override public CombineFileSplit[] getSplits(JobConf job, int numSplits) throws IOException { - long minSize = job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), 0); + long minSize = job.getLong("mapreduce.input.fileinputformat.split.minsize", 0); // For backward compatibility, let the above parameter be used - if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERNODE"), 0) == 0) { + if (job.getLong("mapreduce.input.fileinputformat.split.minsize.per.node", 0) == 0) { super.setMinSplitSizeNode(minSize); } - if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERRACK"), 0) == 0) { + if (job.getLong("mapreduce.input.fileinputformat.split.minsize.per.rack", 0) == 0) { super.setMinSplitSizeRack(minSize); } - if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), 0) == 0) { + if (job.getLong("mapreduce.input.fileinputformat.split.maxsize", 0) == 0) { super.setMaxSplitSize(minSize); }