From 447aa591ef5e9e29870c1936e910d6f6029425c4 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Tue, 22 Jul 2014 12:53:13 -0700 Subject: [PATCH] HIVE-6584 Add HiveHBaseTableSnapshotInputFormat --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 + conf/hive-default.xml.template | 17 ++- .../org/apache/hadoop/hive/hbase/HBaseSplit.java | 104 ++++++++++++-- .../hadoop/hive/hbase/HBaseStorageHandler.java | 55 +++++++- .../hive/hbase/HiveHBaseInputFormatUtil.java | 156 +++++++++++++++++++++ .../hive/hbase/HiveHBaseTableInputFormat.java | 117 +--------------- .../hbase/HiveHBaseTableSnapshotInputFormat.java | 109 ++++++++++++++ .../test/queries/positive/hbase_handler_snapshot.q | 4 + .../test/results/positive/external_table_ppd.q.out | 4 +- .../positive/hbase_binary_storage_queries.q.out | 8 +- .../results/positive/hbase_handler_snapshot.q.out | 13 ++ .../src/test/templates/TestHBaseCliDriver.vm | 1 - .../apache/hadoop/hive/hbase/HBaseQTestUtil.java | 76 +++++++++- .../apache/hadoop/hive/hbase/HBaseTestSetup.java | 82 +++++++---- .../java/org/apache/hadoop/hive/ql/QTestUtil.java | 6 +- pom.xml | 6 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 14 +- 17 files changed, 596 insertions(+), 179 deletions(-) create mode 100644 hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java create mode 100644 hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java create mode 100644 hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q create mode 100644 hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 593c566..5c13fd7 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1252,6 +1252,9 @@ "Disabling this improves HBase write performance at the risk of lost writes in case of a crash."), HIVE_HBASE_GENERATE_HFILES("hive.hbase.generatehfiles", false, "True when HBaseStorageHandler should generate hfiles instead of operate against the online table."), + HIVE_HBASE_SNAPSHOT_NAME("hive.hbase.snapshot.name", null, "The HBase table snapshot name to use."), + HIVE_HBASE_SNAPSHOT_RESTORE_DIR("hive.hbase.snapshot.restoredir", "/tmp", "The directory in which to " + + "restore the HBase table snapshot."), // For har files HIVEARCHIVEENABLED("hive.archive.enabled", false, "Whether archiving operations are permitted"), diff --git a/conf/hive-default.xml.template b/conf/hive-default.xml.template index ba922d0..f7b7808 100644 --- a/conf/hive-default.xml.template +++ b/conf/hive-default.xml.template @@ -1,7 +1,5 @@ - - - - +--> @@ -2208,6 +2205,16 @@ True when HBaseStorageHandler should generate hfiles instead of operate against the online table. + hive.hbase.snapshot.name + + The HBase table snapshot name to use. + + + hive.hbase.snapshot.restoredir + /tmp + The directory in which to restore the HBase table snapshot. + + hive.archive.enabled false Whether archiving operations are permitted diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSplit.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSplit.java index 998c15c..7714bf4 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSplit.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSplit.java @@ -21,6 +21,8 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.mapreduce.TableSplit; @@ -31,46 +33,120 @@ * HBaseSplit augments FileSplit with HBase column mapping. */ public class HBaseSplit extends FileSplit implements InputSplit { - private final TableSplit split; + private static final String TABLESNAPSHOTREGIONSPLIT_CLASS + = "org.apache.hadoop.hbase.mapred.TableSnapshotInputFormat$TableSnapshotRegionSplit"; + + private final TableSplit tableSplit; + private final InputSplit snapshotSplit; + private boolean isTableSplit; // should be final but Writable + + /** + * Create a bare TableSnapshotRegionSplit. Needed because Writables require a + * default-constructed instance to hydrate from the DataInput. + * + * TODO: remove once HBASE-11555 is fixed. + */ + private static InputSplit createTableSnapshotRegionSplit() { + try { + Class resultType = + (Class) Class.forName(TABLESNAPSHOTREGIONSPLIT_CLASS); + Constructor cxtor = resultType.getDeclaredConstructor(new Class[]{}); + cxtor.setAccessible(true); + return cxtor.newInstance(new Object[]{}); + } catch (ClassNotFoundException e) { + throw new UnsupportedOperationException( + "Unable to find " + TABLESNAPSHOTREGIONSPLIT_CLASS, e); + } catch (IllegalAccessException e) { + throw new UnsupportedOperationException( + "Unable to access specified class " + TABLESNAPSHOTREGIONSPLIT_CLASS, e); + } catch (InstantiationException e) { + throw new UnsupportedOperationException( + "Unable to instantiate specified class " + TABLESNAPSHOTREGIONSPLIT_CLASS, e); + } catch (InvocationTargetException e) { + throw new UnsupportedOperationException( + "Constructor threw an exception for " + TABLESNAPSHOTREGIONSPLIT_CLASS, e); + } catch (NoSuchMethodException e) { + throw new UnsupportedOperationException( + "Unable to find suitable constructor for class " + TABLESNAPSHOTREGIONSPLIT_CLASS, e); + } + } + + /** + * For Writable + */ public HBaseSplit() { super((Path) null, 0, 0, (String[]) null); - split = new TableSplit(); + tableSplit = new TableSplit(); + snapshotSplit = createTableSnapshotRegionSplit(); } - public HBaseSplit(TableSplit split, Path dummyPath) { + public HBaseSplit(TableSplit tableSplit, Path dummyPath) { super(dummyPath, 0, 0, (String[]) null); - this.split = split; + this.tableSplit = tableSplit; + this.snapshotSplit = createTableSnapshotRegionSplit(); + this.isTableSplit = true; } - public TableSplit getSplit() { - return this.split; + /** + * TODO: use TableSnapshotRegionSplit HBASE-11555 is fixed. + */ + public HBaseSplit(InputSplit snapshotSplit, Path dummyPath) { + super(dummyPath, 0, 0, (String[]) null); + this.tableSplit = new TableSplit(); + this.snapshotSplit = snapshotSplit; + this.isTableSplit = false; } - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - split.readFields(in); + public TableSplit getTableSplit() { + assert isTableSplit; + return this.tableSplit; + } + + public InputSplit getSnapshotSplit() { + assert !isTableSplit; + return this.snapshotSplit; } @Override public String toString() { - return "TableSplit " + split; + return "" + (isTableSplit ? tableSplit : snapshotSplit); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + this.isTableSplit = in.readBoolean(); + if (this.isTableSplit) { + tableSplit.readFields(in); + } else { + snapshotSplit.readFields(in); + } } @Override public void write(DataOutput out) throws IOException { super.write(out); - split.write(out); + out.writeBoolean(isTableSplit); + if (isTableSplit) { + tableSplit.write(out); + } else { + snapshotSplit.write(out); + } } @Override public long getLength() { - return split.getLength(); + long val = 0; + try { + val = isTableSplit ? tableSplit.getLength() : snapshotSplit.getLength(); + } finally { + return val; + } } @Override public String[] getLocations() throws IOException { - return split.getLocations(); + return isTableSplit ? tableSplit.getLocations() : snapshotSplit.getLocations(); } } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java index dbf5e51..9b86b77 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java @@ -29,7 +29,10 @@ import java.util.Set; import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -38,6 +41,7 @@ import org.apache.hadoop.hbase.mapred.TableOutputFormat; import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormatImpl; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.conf.HiveConf; @@ -69,6 +73,19 @@ public class HBaseStorageHandler extends DefaultStorageHandler implements HiveMetaHook, HiveStoragePredicateHandler { + private static final Log LOG = LogFactory.getLog(HBaseStorageHandler.class); + + /** HBase-internal config by which input format receives snapshot name. */ + private static final String HBASE_SNAPSHOT_NAME_KEY = "hbase.TableSnapshotInputFormat.snapshot.name"; + /** HBase-internal config by which input format received restore dir before HBASE-11335. */ + private static final String HBASE_SNAPSHOT_TABLE_DIR_KEY = "hbase.TableSnapshotInputFormat.table.dir"; + /** HBase-internal config by which input format received restore dir after HBASE-11335. */ + private static final String HBASE_SNAPSHOT_RESTORE_DIR_KEY = "hbase.TableSnapshotInputFormat.restore.dir"; + /** HBase config by which a SlabCache is sized. */ + private static final String HBASE_OFFHEAP_PCT_KEY = "hbase.offheapcache.percentage"; + /** HBase config by which a BucketCache is sized. */ + private static final String HBASE_BUCKETCACHE_SIZE_KEY = "hbase.bucketcache.size"; + final static public String DEFAULT_PREFIX = "default."; //Check if the configure job properties is called from input @@ -258,6 +275,11 @@ public void setConf(Configuration conf) { @Override public Class getInputFormatClass() { + if (HiveConf.getVar(jobConf, HiveConf.ConfVars.HIVE_HBASE_SNAPSHOT_NAME) != null) { + LOG.debug("Using TableSnapshotInputFormat"); + return HiveHBaseTableSnapshotInputFormat.class; + } + LOG.debug("Using HiveHBaseTableInputFormat"); return HiveHBaseTableInputFormat.class; } @@ -342,6 +364,36 @@ public void configureTableJobProperties( // do this for reconciling HBaseStorageHandler for use in HCatalog // check to see if this an input job or an outputjob if (this.configureInputJobProps) { + String snapshotName = HiveConf.getVar(jobConf, HiveConf.ConfVars.HIVE_HBASE_SNAPSHOT_NAME); + if (snapshotName != null) { + try { + // TODO: automatically provide a reasonable restore path when none provided. + String restoreDir = + HiveConf.getVar(jobConf, HiveConf.ConfVars.HIVE_HBASE_SNAPSHOT_RESTORE_DIR); + if (restoreDir == null) { + throw new IllegalArgumentException( + "Cannot process HBase snapshot without specifying " + HiveConf.ConfVars + .HIVE_HBASE_SNAPSHOT_RESTORE_DIR); + } + + TableSnapshotInputFormatImpl.setInput(hbaseConf, snapshotName, new Path(restoreDir)); + // copy over configs touched by above method + jobProperties.put(HBASE_SNAPSHOT_NAME_KEY, hbaseConf.get(HBASE_SNAPSHOT_NAME_KEY)); + if (hbaseConf.get(HBASE_SNAPSHOT_TABLE_DIR_KEY, null) != null) { + jobProperties.put(HBASE_SNAPSHOT_TABLE_DIR_KEY, hbaseConf.get(HBASE_SNAPSHOT_TABLE_DIR_KEY)); + } else { + jobProperties.put(HBASE_SNAPSHOT_RESTORE_DIR_KEY, hbaseConf.get(HBASE_SNAPSHOT_RESTORE_DIR_KEY)); + } + + TableMapReduceUtil.resetCacheConfig(hbaseConf); + // copy over configs touched by above method + jobProperties.put(HBASE_OFFHEAP_PCT_KEY, hbaseConf.get(HBASE_OFFHEAP_PCT_KEY)); + jobProperties.put(HBASE_BUCKETCACHE_SIZE_KEY, hbaseConf.get(HBASE_BUCKETCACHE_SIZE_KEY)); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + for (String k : jobProperties.keySet()) { jobConf.set(k, jobProperties.get(k)); } @@ -415,7 +467,8 @@ public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { * only need TableMapReduceUtil.addDependencyJars(jobConf) here. */ TableMapReduceUtil.addDependencyJars( - jobConf, HBaseStorageHandler.class, TableInputFormatBase.class); + jobConf, HBaseStorageHandler.class, TableInputFormatBase.class, + org.cliffc.high_scale_lib.Counter.class); // this will be removed for HBase 1.0 Set merged = new LinkedHashSet(jobConf.getStringCollection("tmpjars")); Job copy = new Job(jobConf); diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java new file mode 100644 index 0000000..5aa1d79 --- /dev/null +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.hbase; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.mapred.JobConf; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Util code common between HiveHBaseTableInputFormat and HiveHBaseTableSnapshotInputFormat. + */ +class HiveHBaseInputFormatUtil { + + /** + * Parse {@code jobConf} to create the target {@link HTable} instance. + */ + public static HTable getTable(JobConf jobConf) throws IOException { + String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME); + return new HTable(HBaseConfiguration.create(jobConf), Bytes.toBytes(hbaseTableName)); + } + + /** + * Parse {@code jobConf} to create a {@link Scan} instance. + */ + public static Scan getScan(JobConf jobConf) throws IOException { + String hbaseColumnsMapping = jobConf.get(HBaseSerDe.HBASE_COLUMNS_MAPPING); + boolean doColumnRegexMatching = jobConf.getBoolean(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, true); + List readColIDs = ColumnProjectionUtils.getReadColumnIDs(jobConf); + ColumnMappings columnMappings; + + try { + columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping, doColumnRegexMatching); + } catch (SerDeException e) { + throw new IOException(e); + } + + if (columnMappings.size() < readColIDs.size()) { + throw new IOException("Cannot read more columns than the given table contains."); + } + + boolean readAllColumns = ColumnProjectionUtils.isReadAllColumns(jobConf); + Scan scan = new Scan(); + boolean empty = true; + + // The list of families that have been added to the scan + List addedFamilies = new ArrayList(); + + if (!readAllColumns) { + ColumnMapping[] columnsMapping = columnMappings.getColumnsMapping(); + for (int i : readColIDs) { + ColumnMapping colMap = columnsMapping[i]; + if (colMap.hbaseRowKey) { + continue; + } + + if (colMap.qualifierName == null) { + scan.addFamily(colMap.familyNameBytes); + addedFamilies.add(colMap.familyName); + } else { + if(!addedFamilies.contains(colMap.familyName)){ + // add only if the corresponding family has not already been added + scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes); + } + } + + empty = false; + } + } + + // The HBase table's row key maps to a Hive table column. In the corner case when only the + // row key column is selected in Hive, the HBase Scan will be empty i.e. no column family/ + // column qualifier will have been added to the scan. We arbitrarily add at least one column + // to the HBase scan so that we can retrieve all of the row keys and return them as the Hive + // tables column projection. + if (empty) { + for (ColumnMapping colMap: columnMappings) { + if (colMap.hbaseRowKey) { + continue; + } + + if (colMap.qualifierName == null) { + scan.addFamily(colMap.familyNameBytes); + } else { + scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes); + } + + if (!readAllColumns) { + break; + } + } + } + + String scanCache = jobConf.get(HBaseSerDe.HBASE_SCAN_CACHE); + if (scanCache != null) { + scan.setCaching(Integer.valueOf(scanCache)); + } + String scanCacheBlocks = jobConf.get(HBaseSerDe.HBASE_SCAN_CACHEBLOCKS); + if (scanCacheBlocks != null) { + scan.setCacheBlocks(Boolean.valueOf(scanCacheBlocks)); + } + String scanBatch = jobConf.get(HBaseSerDe.HBASE_SCAN_BATCH); + if (scanBatch != null) { + scan.setBatch(Integer.valueOf(scanBatch)); + } + return scan; + } + + public static boolean getStorageFormatOfKey(String spec, String defaultFormat) throws IOException{ + + String[] mapInfo = spec.split("#"); + boolean tblLevelDefault = "binary".equalsIgnoreCase(defaultFormat); + + switch (mapInfo.length) { + case 1: + return tblLevelDefault; + + case 2: + String storageType = mapInfo[1]; + if(storageType.equals("-")) { + return tblLevelDefault; + } else if ("string".startsWith(storageType)){ + return false; + } else if ("binary".startsWith(storageType)){ + return true; + } + + default: + throw new IOException("Malformed string: " + spec); + } + } +} diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java index 1032cc9..4ac0803 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.ByteStream; -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; @@ -88,90 +87,11 @@ final Reporter reporter) throws IOException { HBaseSplit hbaseSplit = (HBaseSplit) split; - TableSplit tableSplit = hbaseSplit.getSplit(); - String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME); - setHTable(new HTable(HBaseConfiguration.create(jobConf), Bytes.toBytes(hbaseTableName))); - String hbaseColumnsMapping = jobConf.get(HBaseSerDe.HBASE_COLUMNS_MAPPING); - boolean doColumnRegexMatching = jobConf.getBoolean(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, true); - List readColIDs = ColumnProjectionUtils.getReadColumnIDs(jobConf); - ColumnMappings columnMappings; - - try { - columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping, doColumnRegexMatching); - } catch (SerDeException e) { - throw new IOException(e); - } - - if (columnMappings.size() < readColIDs.size()) { - throw new IOException("Cannot read more columns than the given table contains."); - } - - boolean readAllColumns = ColumnProjectionUtils.isReadAllColumns(jobConf); - Scan scan = new Scan(); - boolean empty = true; - - // The list of families that have been added to the scan - List addedFamilies = new ArrayList(); - - if (!readAllColumns) { - ColumnMapping[] columnsMapping = columnMappings.getColumnsMapping(); - for (int i : readColIDs) { - ColumnMapping colMap = columnsMapping[i]; - if (colMap.hbaseRowKey) { - continue; - } - - if (colMap.qualifierName == null) { - scan.addFamily(colMap.familyNameBytes); - addedFamilies.add(colMap.familyName); - } else { - if(!addedFamilies.contains(colMap.familyName)){ - // add only if the corresponding family has not already been added - scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes); - } - } - - empty = false; - } - } - - // The HBase table's row key maps to a Hive table column. In the corner case when only the - // row key column is selected in Hive, the HBase Scan will be empty i.e. no column family/ - // column qualifier will have been added to the scan. We arbitrarily add at least one column - // to the HBase scan so that we can retrieve all of the row keys and return them as the Hive - // tables column projection. - if (empty) { - for (ColumnMapping colMap: columnMappings) { - if (colMap.hbaseRowKey) { - continue; - } - - if (colMap.qualifierName == null) { - scan.addFamily(colMap.familyNameBytes); - } else { - scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes); - } + TableSplit tableSplit = hbaseSplit.getTableSplit(); - if (!readAllColumns) { - break; - } - } - } - - String scanCache = jobConf.get(HBaseSerDe.HBASE_SCAN_CACHE); - if (scanCache != null) { - scan.setCaching(Integer.valueOf(scanCache)); - } - String scanCacheBlocks = jobConf.get(HBaseSerDe.HBASE_SCAN_CACHEBLOCKS); - if (scanCacheBlocks != null) { - scan.setCacheBlocks(Boolean.valueOf(scanCacheBlocks)); - } - String scanBatch = jobConf.get(HBaseSerDe.HBASE_SCAN_BATCH); - if (scanBatch != null) { - scan.setBatch(Integer.valueOf(scanBatch)); - } + setHTable(HiveHBaseInputFormatUtil.getTable(jobConf)); + setScan(HiveHBaseInputFormatUtil.getScan(jobConf)); - setScan(scan); Job job = new Job(jobConf); TaskAttemptContext tac = ShimLoader.getHadoopShims().newTaskAttemptContext( job.getConfiguration(), reporter); @@ -443,12 +363,12 @@ static IndexPredicateAnalyzer newIndexPredicateAnalyzer( boolean doColumnRegexMatching = jobConf.getBoolean(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, true); if (hbaseColumnsMapping == null) { - throw new IOException("hbase.columns.mapping required for HBase Table."); + throw new IOException(HBaseSerDe.HBASE_COLUMNS_MAPPING + " required for HBase Table."); } ColumnMappings columnMappings = null; try { - columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping,doColumnRegexMatching); + columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping, doColumnRegexMatching); } catch (SerDeException e) { throw new IOException(e); } @@ -463,10 +383,9 @@ static IndexPredicateAnalyzer newIndexPredicateAnalyzer( // definition into account and excludes regions which don't satisfy // the start/stop row conditions (HBASE-1829). Scan scan = createFilterScan(jobConf, iKey, - getStorageFormatOfKey(keyMapping.mappingSpec, + HiveHBaseInputFormatUtil.getStorageFormatOfKey(keyMapping.mappingSpec, jobConf.get(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "string"))); - // The list of families that have been added to the scan List addedFamilies = new ArrayList(); @@ -503,28 +422,4 @@ static IndexPredicateAnalyzer newIndexPredicateAnalyzer( return results; } - - private boolean getStorageFormatOfKey(String spec, String defaultFormat) throws IOException{ - - String[] mapInfo = spec.split("#"); - boolean tblLevelDefault = "binary".equalsIgnoreCase(defaultFormat) ? true : false; - - switch (mapInfo.length) { - case 1: - return tblLevelDefault; - - case 2: - String storageType = mapInfo[1]; - if(storageType.equals("-")) { - return tblLevelDefault; - } else if ("string".startsWith(storageType)){ - return false; - } else if ("binary".startsWith(storageType)){ - return true; - } - - default: - throw new IOException("Malformed string: " + spec); - } - } } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java new file mode 100644 index 0000000..45e4de9 --- /dev/null +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.hbase; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapred.TableInputFormat; +import org.apache.hadoop.hbase.mapred.TableSnapshotInputFormat; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; + +import java.io.IOException; +import java.util.List; + +public class HiveHBaseTableSnapshotInputFormat + implements InputFormat { + + TableSnapshotInputFormat delegate = new TableSnapshotInputFormat(); + + private static void setColumns(JobConf job) throws IOException { + // hbase mapred API doesn't support scan at the moment. + Scan scan = HiveHBaseInputFormatUtil.getScan(job); + byte[][] families = scan.getFamilies(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < families.length; i++) { + if (i > 0) sb.append(" "); + sb.append(Bytes.toString(families[i])); + } + job.set(TableInputFormat.COLUMN_LIST, sb.toString()); + } + + @Override + public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { + setColumns(job); + + // hive depends on FileSplits, so wrap in HBaseSplit + Path[] tablePaths = FileInputFormat.getInputPaths(job); + + InputSplit [] results = delegate.getSplits(job, numSplits); + for (int i = 0; i < results.length; i++) { + results[i] = new HBaseSplit(results[i], tablePaths[0]); + } + + return results; + } + + @Override + public RecordReader getRecordReader( + InputSplit split, JobConf job, Reporter reporter) throws IOException { + setColumns(job); + final RecordReader rr = + delegate.getRecordReader(((HBaseSplit) split).getSnapshotSplit(), job, reporter); + + return new RecordReader() { + @Override + public boolean next(ImmutableBytesWritable key, ResultWritable value) throws IOException { + return rr.next(key, value.getResult()); + } + + @Override + public ImmutableBytesWritable createKey() { + return rr.createKey(); + } + + @Override + public ResultWritable createValue() { + return new ResultWritable(rr.createValue()); + } + + @Override + public long getPos() throws IOException { + return rr.getPos(); + } + + @Override + public void close() throws IOException { + rr.close(); + } + + @Override + public float getProgress() throws IOException { + return rr.getProgress(); + } + }; + } +} diff --git a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q new file mode 100644 index 0000000..11d52fd --- /dev/null +++ b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q @@ -0,0 +1,4 @@ +SET hive.hbase.snapshot.name=src_hbase_snapshot; +SET hive.hbase.snapshot.restoredir=/tmp; + +SELECT * FROM src_hbase LIMIT 5; diff --git a/hbase-handler/src/test/results/positive/external_table_ppd.q.out b/hbase-handler/src/test/results/positive/external_table_ppd.q.out index 6f1adf4..3519279 100644 --- a/hbase-handler/src/test/results/positive/external_table_ppd.q.out +++ b/hbase-handler/src/test/results/positive/external_table_ppd.q.out @@ -63,8 +63,8 @@ Table Parameters: # Storage Information SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe -InputFormat: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat +InputFormat: null +OutputFormat: null Compressed: No Num Buckets: -1 Bucket Columns: [] diff --git a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out index b92db11..edaa88d 100644 --- a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out +++ b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out @@ -63,8 +63,8 @@ Table Parameters: # Storage Information SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe -InputFormat: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat +InputFormat: null +OutputFormat: null Compressed: No Num Buckets: -1 Bucket Columns: [] @@ -238,8 +238,8 @@ Table Parameters: # Storage Information SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe -InputFormat: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat +InputFormat: null +OutputFormat: null Compressed: No Num Buckets: -1 Bucket Columns: [] diff --git a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out new file mode 100644 index 0000000..1cb18b2 --- /dev/null +++ b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: SELECT * FROM src_hbase LIMIT 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_hbase +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src_hbase LIMIT 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_hbase +#### A masked pattern was here #### +0 val_0 +10 val_10 +100 val_100 +103 val_103 +104 val_104 diff --git a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm index 01d596a..4c05e5b 100644 --- a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm +++ b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm @@ -27,7 +27,6 @@ import java.util.*; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; -import org.apache.hadoop.hive.ql.session.SessionState; public class $className extends TestCase { diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java index 96a0de2..7e6377d 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java @@ -17,24 +17,98 @@ */ package org.apache.hadoop.hive.hbase; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.QTestUtil; -import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; + +import java.util.List; /** * HBaseQTestUtil initializes HBase-specific test fixtures. */ public class HBaseQTestUtil extends QTestUtil { + + /** Name of the HBase table, in both Hive and HBase. */ + public static String HBASE_SRC_NAME = "src_hbase"; + + /** Name of the table snapshot. */ + public static String HBASE_SRC_SNAPSHOT_NAME = "src_hbase_snapshot"; + + /** A handle to this harness's cluster */ + private final HConnection conn; + public HBaseQTestUtil( String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup) throws Exception { super(outDir, logDir, miniMr, null); setup.preTest(conf); + this.conn = setup.getConnection(); super.init(); } + /** return true when HBase table snapshot exists, false otherwise. */ + private static boolean hbaseTableSnapshotExists(HBaseAdmin admin, String snapshotName) throws + Exception { + List snapshots = + admin.listSnapshots(".*" + snapshotName + ".*"); + for (HBaseProtos.SnapshotDescription sn : snapshots) { + if (sn.getName().equals(HBASE_SRC_SNAPSHOT_NAME)) { + return true; + } + } + return false; + } + @Override public void init() throws Exception { // defer } + + @Override + public void createSources() throws Exception { + super.createSources(); + + conf.setBoolean("hive.test.init.phase", true); + + // create and load the input data into the hbase table + runCreateTableCmd( + "CREATE TABLE " + HBASE_SRC_NAME + "(key INT, value STRING)" + + " STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping' = ':key,cf:val')" + + " TBLPROPERTIES ('hbase.table.name' = '" + HBASE_SRC_NAME + "')" + ); + runCmd("INSERT OVERWRITE TABLE " + HBASE_SRC_NAME + " SELECT * FROM src"); + + // create a snapshot + HBaseAdmin admin = null; + try { + admin = new HBaseAdmin(conn.getConfiguration()); + admin.snapshot(HBASE_SRC_SNAPSHOT_NAME, HBASE_SRC_NAME); + } finally { + if (admin != null) admin.close(); + } + + conf.setBoolean("hive.test.init.phase", false); + } + + @Override + public void cleanUp() throws Exception { + super.cleanUp(); + + // drop in case leftover from unsuccessful run + db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, HBASE_SRC_NAME); + + HBaseAdmin admin = null; + try { + admin = new HBaseAdmin(conn.getConfiguration()); + if (hbaseTableSnapshotExists(admin, HBASE_SRC_SNAPSHOT_NAME)) { + admin.deleteSnapshot(HBASE_SRC_SNAPSHOT_NAME); + } + } finally { + if (admin != null) admin.close(); + } + } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java index cdc0a65..300f1cf 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.hbase; -import java.io.File; import java.io.IOException; import java.net.ServerSocket; import java.util.Arrays; @@ -29,12 +28,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.conf.HiveConf; @@ -50,6 +50,7 @@ private MiniHBaseCluster hbaseCluster; private int zooKeeperPort; private String hbaseRoot; + private HConnection hbaseConn; private static final int NUM_REGIONSERVERS = 1; @@ -57,6 +58,10 @@ public HBaseTestSetup(Test test) { super(test); } + public HConnection getConnection() { + return this.hbaseConn; + } + void preTest(HiveConf conf) throws Exception { setUpFixtures(conf); @@ -97,27 +102,23 @@ private void setUpFixtures(HiveConf conf) throws Exception { hbaseConf.setInt("hbase.regionserver.info.port", -1); hbaseCluster = new MiniHBaseCluster(hbaseConf, NUM_REGIONSERVERS); conf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort()); + hbaseConn = HConnectionManager.createConnection(hbaseConf); + // opening the META table ensures that cluster is running - new HTable(hbaseConf, HConstants.META_TABLE_NAME); - createHBaseTable(hbaseConf); + HTableInterface meta = null; + try { + meta = hbaseConn.getTable(TableName.META_TABLE_NAME); + } finally { + if (meta != null) meta.close(); + } + createHBaseTable(); } - private void createHBaseTable(Configuration hbaseConf) throws IOException { + private void createHBaseTable() throws IOException { final String HBASE_TABLE_NAME = "HiveExternalTable"; HTableDescriptor htableDesc = new HTableDescriptor(HBASE_TABLE_NAME.getBytes()); HColumnDescriptor hcolDesc = new HColumnDescriptor("cf".getBytes()); htableDesc.addFamily(hcolDesc); - HBaseAdmin hbaseAdmin = new HBaseAdmin(hbaseConf); - if(Arrays.asList(hbaseAdmin.listTables()).contains(htableDesc)){ - // if table is already in there, don't recreate. - return; - } - hbaseAdmin.createTable(htableDesc); - HTable htable = new HTable(hbaseConf, HBASE_TABLE_NAME); - - // data - Put [] puts = new Put [] { - new Put("key-1".getBytes()), new Put("key-2".getBytes()), new Put("key-3".getBytes()) }; boolean [] booleans = new boolean [] { true, false, true }; byte [] bytes = new byte [] { Byte.MIN_VALUE, -1, Byte.MAX_VALUE }; @@ -128,18 +129,37 @@ private void createHBaseTable(Configuration hbaseConf) throws IOException { float [] floats = new float [] { Float.MIN_VALUE, -1.0F, Float.MAX_VALUE }; double [] doubles = new double [] { Double.MIN_VALUE, -1.0, Double.MAX_VALUE }; - // store data - for (int i = 0; i < puts.length; i++) { - puts[i].add("cf".getBytes(), "cq-boolean".getBytes(), Bytes.toBytes(booleans[i])); - puts[i].add("cf".getBytes(), "cq-byte".getBytes(), new byte [] { bytes[i] }); - puts[i].add("cf".getBytes(), "cq-short".getBytes(), Bytes.toBytes(shorts[i])); - puts[i].add("cf".getBytes(), "cq-int".getBytes(), Bytes.toBytes(ints[i])); - puts[i].add("cf".getBytes(), "cq-long".getBytes(), Bytes.toBytes(longs[i])); - puts[i].add("cf".getBytes(), "cq-string".getBytes(), Bytes.toBytes(strings[i])); - puts[i].add("cf".getBytes(), "cq-float".getBytes(), Bytes.toBytes(floats[i])); - puts[i].add("cf".getBytes(), "cq-double".getBytes(), Bytes.toBytes(doubles[i])); - - htable.put(puts[i]); + HBaseAdmin hbaseAdmin = null; + HTableInterface htable = null; + try { + hbaseAdmin = new HBaseAdmin(hbaseConn.getConfiguration()); + if (Arrays.asList(hbaseAdmin.listTables()).contains(htableDesc)) { + // if table is already in there, don't recreate. + return; + } + hbaseAdmin.createTable(htableDesc); + htable = hbaseConn.getTable(HBASE_TABLE_NAME); + + // data + Put[] puts = new Put[]{ + new Put("key-1".getBytes()), new Put("key-2".getBytes()), new Put("key-3".getBytes())}; + + // store data + for (int i = 0; i < puts.length; i++) { + puts[i].add("cf".getBytes(), "cq-boolean".getBytes(), Bytes.toBytes(booleans[i])); + puts[i].add("cf".getBytes(), "cq-byte".getBytes(), new byte[]{bytes[i]}); + puts[i].add("cf".getBytes(), "cq-short".getBytes(), Bytes.toBytes(shorts[i])); + puts[i].add("cf".getBytes(), "cq-int".getBytes(), Bytes.toBytes(ints[i])); + puts[i].add("cf".getBytes(), "cq-long".getBytes(), Bytes.toBytes(longs[i])); + puts[i].add("cf".getBytes(), "cq-string".getBytes(), Bytes.toBytes(strings[i])); + puts[i].add("cf".getBytes(), "cq-float".getBytes(), Bytes.toBytes(floats[i])); + puts[i].add("cf".getBytes(), "cq-double".getBytes(), Bytes.toBytes(doubles[i])); + + htable.put(puts[i]); + } + } finally { + if (htable != null) htable.close(); + if (hbaseAdmin != null) hbaseAdmin.close(); } } @@ -152,6 +172,10 @@ private static int findFreePort() throws IOException { @Override protected void tearDown() throws Exception { + if (hbaseConn != null) { + hbaseConn.close(); + hbaseConn = null; + } if (hbaseCluster != null) { HConnectionManager.deleteAllConnections(true); hbaseCluster.shutdown(); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 2fefa06..6478199 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -130,7 +130,7 @@ public static final HashSet srcTables = new HashSet(); private static MiniClusterType clusterType = MiniClusterType.none; private ParseDriver pd; - private Hive db; + protected Hive db; protected HiveConf conf; private Driver drv; private BaseSemanticAnalyzer sem; @@ -630,7 +630,7 @@ private void runLoadCmd(String loadCmd) throws Exception { return; } - private void runCreateTableCmd(String createTableCmd) throws Exception { + protected void runCreateTableCmd(String createTableCmd) throws Exception { int ecode = 0; ecode = drv.run(createTableCmd).getResponseCode(); if (ecode != 0) { @@ -641,7 +641,7 @@ private void runCreateTableCmd(String createTableCmd) throws Exception { return; } - private void runCmd(String cmd) throws Exception { + protected void runCmd(String cmd) throws Exception { int ecode = 0; ecode = drv.run(cmd).getResponseCode(); drv.close(); diff --git a/pom.xml b/pom.xml index b5a5697..614cae4 100644 --- a/pom.xml +++ b/pom.xml @@ -111,8 +111,8 @@ 1.2.1 2.4.0 ${basedir}/${hive.path.to.root}/testutils/hadoop - 0.96.0-hadoop1 - 0.96.0-hadoop2 + 0.98.3-hadoop1 + 0.98.3-hadoop2 4.2.5 4.2.5 @@ -772,7 +772,7 @@ ${test.warehouse.scheme}${test.warehouse.dir} true - src,src1,srcbucket,srcbucket2,src_json,src_thrift,src_sequencefile,srcpart,alltypesorc + src,src1,srcbucket,srcbucket2,src_json,src_thrift,src_sequencefile,srcpart,alltypesorc,src_hbase ${test.tmp.dir}/conf/krb5.conf diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index c80a2a3..33e29ab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -3919,12 +3919,16 @@ private int createTable(Hive db, CreateTableDesc crtTbl) throws HiveException { tbl.setInputFormatClass(crtTbl.getInputFormat()); tbl.setOutputFormatClass(crtTbl.getOutputFormat()); - tbl.getTTable().getSd().setInputFormat( - tbl.getInputFormatClass().getName()); - tbl.getTTable().getSd().setOutputFormat( - tbl.getOutputFormatClass().getName()); + // only persist input/ouput format to metadata when it is explicitly specified. + // Otherwise, load lazily via StorageHandler at query time. + if (crtTbl.getInputFormat() != null && !crtTbl.getInputFormat().isEmpty()) { + tbl.getTTable().getSd().setInputFormat(tbl.getInputFormatClass().getName()); + } + if (crtTbl.getOutputFormat() != null && !crtTbl.getOutputFormat().isEmpty()) { + tbl.getTTable().getSd().setOutputFormat(tbl.getOutputFormatClass().getName()); + } - if (!Utilities.isDefaultNameNode(conf)) { + if (!Utilities.isDefaultNameNode(conf) && tbl.getTTable().getSd().isSetLocation()) { // If location is specified - ensure that it is a full qualified name makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tbl.getTableName()); } -- 1.9.0