diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 5bdeb92..e4cbb0c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -2354,6 +2354,32 @@ public static void copyTableJobPropertiesToConf(TableDesc tbl, JobConf job) { } } + /** + * Copies the storage handler proeprites configured for a table descriptor to a runtime job + * configuration. This differs from {@link #copyTablePropertiesToConf(org.apache.hadoop.hive.ql.plan.TableDesc, org.apache.hadoop.mapred.JobConf)} + * in that it does not allow parameters already set in the job to override the values from the + * table. This is important for setting the config up for reading, + * as the job may already have values in it from another table. + * @param tbl + * @param job + */ + public static void copyTablePropertiesToConf(TableDesc tbl, JobConf job) { + Properties tblProperties = tbl.getProperties(); + for(String name: tblProperties.stringPropertyNames()) { + String val = (String) tblProperties.get(name); + if (val != null) { + job.set(name, StringEscapeUtils.escapeJava(val)); + } + } + Map jobProperties = tbl.getJobProperties(); + if (jobProperties == null) { + return; + } + for (Map.Entry entry : jobProperties.entrySet()) { + job.set(entry.getKey(), entry.getValue()); + } + } + private static final Object INPUT_SUMMARY_LOCK = new Object(); /** diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index 13c9751..a6a8176 100755 --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -275,7 +275,7 @@ private void addSplitsForGroup(List dirs, TableScanOperator tableScan, Job InputFormat inputFormat, Class inputFormatClass, int splits, TableDesc table, List result) throws IOException { - Utilities.copyTableJobPropertiesToConf(table, conf); + Utilities.copyTablePropertiesToConf(table, conf); if (tableScan != null) { pushFilters(conf, tableScan); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 7c2def7..9340ce6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -388,6 +388,7 @@ public boolean validateInput(FileSystem fs, HiveConf conf, ConfVars.HIVE_ORC_INCLUDE_FILE_FOOTER_IN_SPLITS); numBuckets = Math.max(conf.getInt(hive_metastoreConstants.BUCKET_COUNT, 0), 0); + LOG.debug("Number of buckets specified by conf file is " + numBuckets); int cacheStripeDetailsSize = HiveConf.getIntVar(conf, ConfVars.HIVE_ORC_CACHE_STRIPE_DETAILS_SIZE); int numThreads = HiveConf.getIntVar(conf,