diff --git itests/qtest/pom.xml itests/qtest/pom.xml index 1b49e88..1c3b601 100644 --- itests/qtest/pom.xml +++ itests/qtest/pom.xml @@ -119,6 +119,13 @@ tests test + + org.apache.hive + hive-jdbc-handler + ${project.version} + test + + diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 5b30157..c68a366 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -498,6 +498,7 @@ minillaplocal.query.files=acid_globallimit.q,\ input16_cc.q,\ insert_dir_distcp.q,\ insert_into_with_schema.q,\ + jdbc_handler.q,\ join1.q,\ join_acid_non_acid.q,\ join_filters.q,\ diff --git jdbc-handler/pom.xml jdbc-handler/pom.xml new file mode 100644 index 0000000..364886a --- /dev/null +++ jdbc-handler/pom.xml @@ -0,0 +1,127 @@ + + + + 4.0.0 + + org.apache.hive + hive + 2.2.0-SNAPSHOT + ../pom.xml + + + hive-jdbc-handler + jar + Hive JDBC Handler + + + .. + + + + + org.apache.hive + hive-common + ${project.version} + + + org.eclipse.jetty.aggregate + jetty-all + + + + + + org.apache.hive + hive-shims + ${project.version} + + + + org.apache.hive + hive-exec + ${project.version} + + + + org.apache.hive + hive-serde + ${project.version} + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + true + + + org.apache.hadoop + hadoop-mapreduce-client-common + ${hadoop.version} + true + test + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + + + + org.hamcrest + hamcrest-all + ${hamcrest.version} + test + + + + junit + junit + ${junit.version} + test + + + + org.mockito + mockito-all + ${mockito-all.version} + test + + + + org.apache.hive + hive-common + ${project.version} + test + test-jar + + + + com.h2database + h2 + ${h2database.version} + test + + + + + diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcInputFormat.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcInputFormat.java new file mode 100644 index 0000000..bfa7a26 --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcInputFormat.java @@ -0,0 +1,108 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.storage.jdbc; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.HiveInputFormat; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.MapWritable; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hive.storage.jdbc.dao.DatabaseAccessor; +import org.apache.hive.storage.jdbc.dao.DatabaseAccessorFactory; + +import java.io.IOException; + +public class JdbcInputFormat extends HiveInputFormat { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcInputFormat.class); + private DatabaseAccessor dbAccessor = null; + + + /** + * {@inheritDoc} + */ + @Override + public RecordReader + getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { + + if (!(split instanceof JdbcInputSplit)) { + throw new RuntimeException("Incompatible split type " + split.getClass().getName() + "."); + } + + return new JdbcRecordReader(job, (JdbcInputSplit) split); + } + + + /** + * {@inheritDoc} + */ + @Override + public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { + try { + if (numSplits <= 0) { + numSplits = 1; + } + LOGGER.debug("Creating {} input splits", numSplits); + if (dbAccessor == null) { + dbAccessor = DatabaseAccessorFactory.getAccessor(job); + } + + int numRecords = dbAccessor.getTotalNumberOfRecords(job); + int numRecordsPerSplit = numRecords / numSplits; + int numSplitsWithExtraRecords = numRecords % numSplits; + + LOGGER.debug("Num records = {}", numRecords); + InputSplit[] splits = new InputSplit[numSplits]; + Path[] tablePaths = FileInputFormat.getInputPaths(job); + + int offset = 0; + for (int i = 0; i < numSplits; i++) { + int numRecordsInThisSplit = numRecordsPerSplit; + if (i < numSplitsWithExtraRecords) { + numRecordsInThisSplit++; + } + + splits[i] = new JdbcInputSplit(numRecordsInThisSplit, offset, tablePaths[0]); + offset += numRecordsInThisSplit; + } + + return splits; + } + catch (Exception e) { + LOGGER.error("Error while splitting input data.", e); + throw new IOException(e); + } + } + + + /** + * For testing purposes only + * + * @param dbAccessor + * DatabaseAccessor object + */ + public void setDbAccessor(DatabaseAccessor dbAccessor) { + this.dbAccessor = dbAccessor; + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcInputSplit.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcInputSplit.java new file mode 100644 index 0000000..a691cc2 --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcInputSplit.java @@ -0,0 +1,100 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class JdbcInputSplit extends FileSplit implements InputSplit { + + private static final String[] EMPTY_ARRAY = new String[] {}; + + private int limit = 0; + private int offset = 0; + + + public JdbcInputSplit() { + super((Path) null, 0, 0, EMPTY_ARRAY); + + } + + + public JdbcInputSplit(long start, long end, Path dummyPath) { + super(dummyPath, 0, 0, EMPTY_ARRAY); + this.setLimit((int) start); + this.setOffset((int) end); + } + + + public JdbcInputSplit(int limit, int offset) { + super((Path) null, 0, 0, EMPTY_ARRAY); + this.limit = limit; + this.offset = offset; + } + + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeInt(limit); + out.writeInt(offset); + } + + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + limit = in.readInt(); + offset = in.readInt(); + } + + + @Override + public long getLength() { + return limit; + } + + + @Override + public String[] getLocations() throws IOException { + return EMPTY_ARRAY; + } + + + public int getLimit() { + return limit; + } + + + public void setLimit(int limit) { + this.limit = limit; + } + + + public int getOffset() { + return offset; + } + + + public void setOffset(int offset) { + this.offset = offset; + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcOutputFormat.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcOutputFormat.java new file mode 100644 index 0000000..26fb3cd --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcOutputFormat.java @@ -0,0 +1,68 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.io.MapWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.util.Progressable; + +import java.io.IOException; +import java.util.Properties; + +public class JdbcOutputFormat implements OutputFormat, + HiveOutputFormat { + + /** + * {@inheritDoc} + */ + @Override + public RecordWriter getHiveRecordWriter(JobConf jc, + Path finalOutPath, + Class valueClass, + boolean isCompressed, + Properties tableProperties, + Progressable progress) throws IOException { + throw new UnsupportedOperationException("Write operations are not allowed."); + } + + + /** + * {@inheritDoc} + */ + @Override + public org.apache.hadoop.mapred.RecordWriter getRecordWriter(FileSystem ignored, + JobConf job, + String name, + Progressable progress) throws IOException { + throw new UnsupportedOperationException("Write operations are not allowed."); + } + + + /** + * {@inheritDoc} + */ + @Override + public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { + // do nothing + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcRecordReader.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcRecordReader.java new file mode 100644 index 0000000..0a24bd9 --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcRecordReader.java @@ -0,0 +1,133 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.MapWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hive.storage.jdbc.dao.DatabaseAccessor; +import org.apache.hive.storage.jdbc.dao.DatabaseAccessorFactory; +import org.apache.hive.storage.jdbc.dao.JdbcRecordIterator; + +import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; + +public class JdbcRecordReader implements RecordReader { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcRecordReader.class); + private DatabaseAccessor dbAccessor = null; + private JdbcRecordIterator iterator = null; + private JdbcInputSplit split = null; + private JobConf conf = null; + private int pos = 0; + + + public JdbcRecordReader(JobConf conf, JdbcInputSplit split) { + LOGGER.debug("Initializing JdbcRecordReader"); + this.split = split; + this.conf = conf; + } + + + @Override + public boolean next(LongWritable key, MapWritable value) throws IOException { + try { + LOGGER.debug("JdbcRecordReader.next called"); + if (dbAccessor == null) { + dbAccessor = DatabaseAccessorFactory.getAccessor(conf); + iterator = dbAccessor.getRecordIterator(conf, split.getLimit(), split.getOffset()); + } + + if (iterator.hasNext()) { + LOGGER.debug("JdbcRecordReader has more records to read."); + key.set(pos); + pos++; + Map record = iterator.next(); + if ((record != null) && (!record.isEmpty())) { + for (Entry entry : record.entrySet()) { + value.put(new Text(entry.getKey()), new Text(entry.getValue())); + } + return true; + } + else { + LOGGER.debug("JdbcRecordReader got null record."); + return false; + } + } + else { + LOGGER.debug("JdbcRecordReader has no more records to read."); + return false; + } + } + catch (Exception e) { + LOGGER.error("An error occurred while reading the next record from DB.", e); + return false; + } + } + + + @Override + public LongWritable createKey() { + return new LongWritable(); + } + + + @Override + public MapWritable createValue() { + return new MapWritable(); + } + + + @Override + public long getPos() throws IOException { + return pos; + } + + + @Override + public void close() throws IOException { + if (iterator != null) { + iterator.close(); + } + } + + + @Override + public float getProgress() throws IOException { + if (split == null) { + return 0; + } + else { + return split.getLength() > 0 ? pos / (float) split.getLength() : 1.0f; + } + } + + + public void setDbAccessor(DatabaseAccessor dbAccessor) { + this.dbAccessor = dbAccessor; + } + + + public void setIterator(JdbcRecordIterator iterator) { + this.iterator = iterator; + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java new file mode 100644 index 0000000..f35c33d --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java @@ -0,0 +1,164 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.MapWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfigManager; +import org.apache.hive.storage.jdbc.dao.DatabaseAccessor; +import org.apache.hive.storage.jdbc.dao.DatabaseAccessorFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +public class JdbcSerDe extends AbstractSerDe { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcSerDe.class); + + private StructObjectInspector objectInspector; + private int numColumns; + private String[] hiveColumnTypeArray; + private List columnNames; + private List row; + + + /* + * This method gets called multiple times by Hive. On some invocations, the properties will be empty. + * We need to detect when the properties are not empty to initialise the class variables. + * + * @see org.apache.hadoop.hive.serde2.Deserializer#initialize(org.apache.hadoop.conf.Configuration, java.util.Properties) + */ + @Override + public void initialize(Configuration conf, Properties tbl) throws SerDeException { + try { + LOGGER.debug("Initializing the SerDe"); + + // Hive cdh-4.3 does not provide the properties object on all calls + if (tbl.containsKey(JdbcStorageConfig.DATABASE_TYPE.getPropertyName())) { + Configuration tableConfig = JdbcStorageConfigManager.convertPropertiesToConfiguration(tbl); + + DatabaseAccessor dbAccessor = DatabaseAccessorFactory.getAccessor(tableConfig); + columnNames = dbAccessor.getColumnNames(tableConfig); + numColumns = columnNames.size(); + + String[] hiveColumnNameArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMNS), ","); + if (numColumns != hiveColumnNameArray.length) { + throw new SerDeException("Expected " + numColumns + " columns. Table definition has " + + hiveColumnNameArray.length + " columns"); + } + List hiveColumnNames = Arrays.asList(hiveColumnNameArray); + + hiveColumnTypeArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES), ":"); + if (hiveColumnTypeArray.length == 0) { + throw new SerDeException("Received an empty Hive column type definition"); + } + + List fieldInspectors = new ArrayList(numColumns); + for (int i = 0; i < numColumns; i++) { + fieldInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + } + + objectInspector = + ObjectInspectorFactory.getStandardStructObjectInspector(hiveColumnNames, + fieldInspectors); + row = new ArrayList(numColumns); + } + } + catch (Exception e) { + LOGGER.error("Caught exception while initializing the SqlSerDe", e); + throw new SerDeException(e); + } + } + + + private String[] parseProperty(String propertyValue, String delimiter) { + if ((propertyValue == null) || (propertyValue.trim().isEmpty())) { + return new String[] {}; + } + + return propertyValue.split(delimiter); + } + + + @Override + public Object deserialize(Writable blob) throws SerDeException { + LOGGER.debug("Deserializing from SerDe"); + if (!(blob instanceof MapWritable)) { + throw new SerDeException("Expected MapWritable. Got " + blob.getClass().getName()); + } + + if ((row == null) || (columnNames == null)) { + throw new SerDeException("JDBC SerDe hasn't been initialized properly"); + } + + row.clear(); + MapWritable input = (MapWritable) blob; + Text columnKey = new Text(); + + for (int i = 0; i < numColumns; i++) { + columnKey.set(columnNames.get(i)); + Writable value = input.get(columnKey); + if (value == null) { + row.add(null); + } + else { + row.add(value.toString()); + } + } + + return row; + } + + + @Override + public ObjectInspector getObjectInspector() throws SerDeException { + return objectInspector; + } + + + @Override + public Class getSerializedClass() { + return MapWritable.class; + } + + + @Override + public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { + throw new UnsupportedOperationException("Writes are not allowed"); + } + + + @Override + public SerDeStats getSerDeStats() { + return null; + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcStorageHandler.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcStorageHandler.java new file mode 100644 index 0000000..946ee0c --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcStorageHandler.java @@ -0,0 +1,106 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.HiveMetaHook; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; +import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; + +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfigManager; + +import java.util.Map; +import java.util.Properties; + +public class JdbcStorageHandler implements HiveStorageHandler { + + private Configuration conf; + + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + + @Override + public Configuration getConf() { + return this.conf; + } + + + @SuppressWarnings("rawtypes") + @Override + public Class getInputFormatClass() { + return JdbcInputFormat.class; + } + + + @SuppressWarnings("rawtypes") + @Override + public Class getOutputFormatClass() { + return JdbcOutputFormat.class; + } + + + @Override + public Class getSerDeClass() { + return JdbcSerDe.class; + } + + + @Override + public HiveMetaHook getMetaHook() { + return null; + } + + + @Override + public void configureTableJobProperties(TableDesc tableDesc, Map jobProperties) { + Properties properties = tableDesc.getProperties(); + JdbcStorageConfigManager.copyConfigurationToJob(properties, jobProperties); + } + + + @Override + public void configureInputJobProperties(TableDesc tableDesc, Map jobProperties) { + Properties properties = tableDesc.getProperties(); + JdbcStorageConfigManager.copyConfigurationToJob(properties, jobProperties); + } + + + @Override + public void configureOutputJobProperties(TableDesc tableDesc, Map jobProperties) { + // Nothing to do here... + } + + + @Override + public HiveAuthorizationProvider getAuthorizationProvider() throws HiveException { + return null; + } + + @Override + public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { + + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/QueryConditionBuilder.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/QueryConditionBuilder.java new file mode 100644 index 0000000..194fad8 --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/QueryConditionBuilder.java @@ -0,0 +1,186 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.TableScanDesc; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; + +import java.beans.XMLDecoder; +import java.io.ByteArrayInputStream; +import java.util.HashMap; +import java.util.Map; + +/** + * Translates the hive query condition into a condition that can be run on the underlying database + */ +public class QueryConditionBuilder { + + private static final Logger LOGGER = LoggerFactory.getLogger(QueryConditionBuilder.class); + private static final String EMPTY_STRING = ""; + private static QueryConditionBuilder instance = null; + + + public static QueryConditionBuilder getInstance() { + if (instance == null) { + instance = new QueryConditionBuilder(); + } + + return instance; + } + + + private QueryConditionBuilder() { + + } + + + public String buildCondition(Configuration conf) { + if (conf == null) { + return EMPTY_STRING; + } + + String filterXml = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR); + String hiveColumns = conf.get(serdeConstants.LIST_COLUMNS); + String columnMapping = conf.get(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName()); + + if ((filterXml == null) || ((columnMapping == null) && (hiveColumns == null))) { + return EMPTY_STRING; + } + + if (hiveColumns == null) { + hiveColumns = ""; + } + + Map columnMap = buildColumnMapping(columnMapping, hiveColumns); + String condition = createConditionString(filterXml, columnMap); + return condition; + } + + + /* + * Build a Hive-to-X column mapping, + * + */ + private Map buildColumnMapping(String columnMapping, String hiveColumns) { + if ((columnMapping == null) || (columnMapping.trim().isEmpty())) { + return createIdentityMap(hiveColumns); + } + + Map columnMap = new HashMap(); + String[] mappingPairs = columnMapping.toLowerCase().split(","); + for (String mapPair : mappingPairs) { + String[] columns = mapPair.split("="); + columnMap.put(columns[0].trim(), columns[1].trim()); + } + + return columnMap; + } + + + /* + * When no mapping is defined, it is assumed that the hive column names are equivalent to the column names in the + * underlying table + */ + private Map createIdentityMap(String hiveColumns) { + Map columnMap = new HashMap(); + String[] columns = hiveColumns.toLowerCase().split(","); + + for (String col : columns) { + columnMap.put(col.trim(), col.trim()); + } + + return columnMap; + } + + + /* + * Walk to Hive AST and translate the hive column names to their equivalent mappings. This is basically a cheat. + * + */ + private String createConditionString(String filterXml, Map columnMap) { + if ((filterXml == null) || (filterXml.trim().isEmpty())) { + return EMPTY_STRING; + } + + try (XMLDecoder decoder = new XMLDecoder(new ByteArrayInputStream(filterXml.getBytes("UTF-8")))) { + Object object = decoder.readObject(); + if (!(object instanceof ExprNodeDesc)) { + LOGGER.error("Deserialized filter expression is not of the expected type"); + throw new RuntimeException("Deserialized filter expression is not of the expected type"); + } + + ExprNodeDesc conditionNode = (ExprNodeDesc) object; + walkTreeAndTranslateColumnNames(conditionNode, columnMap); + return conditionNode.getExprString(); + } + catch (Exception e) { + LOGGER.error("Error during condition build", e); + return EMPTY_STRING; + } + } + + + /* + * Translate column names by walking the AST + */ + private void walkTreeAndTranslateColumnNames(ExprNodeDesc node, Map columnMap) { + if (node == null) { + return; + } + + if (node instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc column = (ExprNodeColumnDesc) node; + String hiveColumnName = column.getColumn().toLowerCase(); + if (columnMap.containsKey(hiveColumnName)) { + String dbColumnName = columnMap.get(hiveColumnName); + String finalName = formatColumnName(dbColumnName); + column.setColumn(finalName); + } + } + else { + if (node.getChildren() != null) { + for (ExprNodeDesc childNode : node.getChildren()) { + walkTreeAndTranslateColumnNames(childNode, columnMap); + } + } + } + } + + + /** + * This is an ugly hack for handling date column types because Hive doesn't have a built-in type for dates + */ + private String formatColumnName(String dbColumnName) { + if (dbColumnName.contains(":")) { + String[] typeSplit = dbColumnName.split(":"); + + if (typeSplit[1].equalsIgnoreCase("date")) { + return "{d " + typeSplit[0] + "}"; + } + + return typeSplit[0]; + } + else { + return dbColumnName; + } + } +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/CustomConfigManager.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/CustomConfigManager.java new file mode 100644 index 0000000..7a787d4 --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/CustomConfigManager.java @@ -0,0 +1,23 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.conf; + +import java.util.Properties; + +public interface CustomConfigManager { + + void checkRequiredProperties(Properties properties); + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/CustomConfigManagerFactory.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/CustomConfigManagerFactory.java new file mode 100644 index 0000000..eed0dff --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/CustomConfigManagerFactory.java @@ -0,0 +1,50 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.conf; + +import java.util.Properties; + +/** + * Factory for creating custom config managers based on the database type + */ +public class CustomConfigManagerFactory { + + private static CustomConfigManager nopConfigManager = new NopCustomConfigManager(); + + + private CustomConfigManagerFactory() { + } + + + public static CustomConfigManager getCustomConfigManagerFor(DatabaseType databaseType) { + switch (databaseType) { + case MYSQL: + return nopConfigManager; + + default: + return nopConfigManager; + } + } + + private static class NopCustomConfigManager implements CustomConfigManager { + + @Override + public void checkRequiredProperties(Properties properties) { + return; + } + + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/DatabaseType.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/DatabaseType.java new file mode 100644 index 0000000..a2bdbe4 --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/DatabaseType.java @@ -0,0 +1,21 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.conf; + +public enum DatabaseType { + MYSQL, + H2, + DERBY +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfig.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfig.java new file mode 100644 index 0000000..ff6357d --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfig.java @@ -0,0 +1,49 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.conf; + +public enum JdbcStorageConfig { + DATABASE_TYPE("database.type", true), + JDBC_URL("jdbc.url", true), + JDBC_DRIVER_CLASS("jdbc.driver", true), + QUERY("query", true), + JDBC_FETCH_SIZE("jdbc.fetch.size", false), + COLUMN_MAPPING("column.mapping", false); + + private String propertyName; + private boolean required = false; + + + JdbcStorageConfig(String propertyName, boolean required) { + this.propertyName = propertyName; + this.required = required; + } + + + JdbcStorageConfig(String propertyName) { + this.propertyName = propertyName; + } + + + public String getPropertyName() { + return JdbcStorageConfigManager.CONFIG_PREFIX + "." + propertyName; + } + + + public boolean isRequired() { + return required; + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java new file mode 100644 index 0000000..5267cda --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java @@ -0,0 +1,97 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.conf; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.hive.storage.jdbc.QueryConditionBuilder; + +import java.util.EnumSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; + +/** + * Main configuration handler class + */ +public class JdbcStorageConfigManager { + + public static final String CONFIG_PREFIX = "hive.sql"; + private static final EnumSet DEFAULT_REQUIRED_PROPERTIES = + EnumSet.of(JdbcStorageConfig.DATABASE_TYPE, + JdbcStorageConfig.JDBC_URL, + JdbcStorageConfig.JDBC_DRIVER_CLASS, + JdbcStorageConfig.QUERY); + + + private JdbcStorageConfigManager() { + } + + + public static void copyConfigurationToJob(Properties props, Map jobProps) { + checkRequiredPropertiesAreDefined(props); + for (Entry entry : props.entrySet()) { + jobProps.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue())); + } + } + + + public static Configuration convertPropertiesToConfiguration(Properties props) { + checkRequiredPropertiesAreDefined(props); + Configuration conf = new Configuration(); + + for (Entry entry : props.entrySet()) { + conf.set(String.valueOf(entry.getKey()), String.valueOf(entry.getValue())); + } + + return conf; + } + + + private static void checkRequiredPropertiesAreDefined(Properties props) { + for (JdbcStorageConfig configKey : DEFAULT_REQUIRED_PROPERTIES) { + String propertyKey = configKey.getPropertyName(); + if ((props == null) || (!props.containsKey(propertyKey)) || (isEmptyString(props.getProperty(propertyKey)))) { + throw new IllegalArgumentException("Property " + propertyKey + " is required."); + } + } + + DatabaseType dbType = DatabaseType.valueOf(props.getProperty(JdbcStorageConfig.DATABASE_TYPE.getPropertyName())); + CustomConfigManager configManager = CustomConfigManagerFactory.getCustomConfigManagerFor(dbType); + configManager.checkRequiredProperties(props); + } + + + public static String getConfigValue(JdbcStorageConfig key, Configuration config) { + return config.get(key.getPropertyName()); + } + + + public static String getQueryToExecute(Configuration config) { + String query = config.get(JdbcStorageConfig.QUERY.getPropertyName()); + String hiveFilterCondition = QueryConditionBuilder.getInstance().buildCondition(config); + if ((hiveFilterCondition != null) && (!hiveFilterCondition.trim().isEmpty())) { + query = query + " WHERE " + hiveFilterCondition; + } + + return query; + } + + + private static boolean isEmptyString(String value) { + return ((value == null) || (value.trim().isEmpty())); + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java new file mode 100644 index 0000000..f50d53e --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java @@ -0,0 +1,34 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.dao; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException; + +import java.util.List; + +public interface DatabaseAccessor { + + List getColumnNames(Configuration conf) throws HiveJdbcDatabaseAccessException; + + + int getTotalNumberOfRecords(Configuration conf) throws HiveJdbcDatabaseAccessException; + + + JdbcRecordIterator + getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException; + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessorFactory.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessorFactory.java new file mode 100644 index 0000000..7dc690f --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessorFactory.java @@ -0,0 +1,53 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.dao; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.hive.storage.jdbc.conf.DatabaseType; +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; + +/** + * Factory for creating the correct DatabaseAccessor class for the job + */ +public class DatabaseAccessorFactory { + + private DatabaseAccessorFactory() { + } + + + public static DatabaseAccessor getAccessor(DatabaseType dbType) { + + DatabaseAccessor accessor = null; + switch (dbType) { + case MYSQL: + accessor = new MySqlDatabaseAccessor(); + break; + + default: + accessor = new GenericJdbcDatabaseAccessor(); + break; + } + + return accessor; + } + + + public static DatabaseAccessor getAccessor(Configuration conf) { + DatabaseType dbType = DatabaseType.valueOf(conf.get(JdbcStorageConfig.DATABASE_TYPE.getPropertyName())); + return getAccessor(dbType); + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java new file mode 100644 index 0000000..b655aec --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java @@ -0,0 +1,253 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.dao; + +import org.apache.commons.dbcp.BasicDataSourceFactory; +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfigManager; +import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException; + +import javax.sql.DataSource; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; + +/** + * A data accessor that should in theory work with all JDBC compliant database drivers. + */ +public class GenericJdbcDatabaseAccessor implements DatabaseAccessor { + + protected static final String DBCP_CONFIG_PREFIX = JdbcStorageConfigManager.CONFIG_PREFIX + ".dbcp"; + protected static final int DEFAULT_FETCH_SIZE = 1000; + protected static final Logger LOGGER = LoggerFactory.getLogger(GenericJdbcDatabaseAccessor.class); + protected DataSource dbcpDataSource = null; + + + public GenericJdbcDatabaseAccessor() { + } + + + @Override + public List getColumnNames(Configuration conf) throws HiveJdbcDatabaseAccessException { + Connection conn = null; + PreparedStatement ps = null; + ResultSet rs = null; + + try { + initializeDatabaseConnection(conf); + String sql = JdbcStorageConfigManager.getQueryToExecute(conf); + String metadataQuery = addLimitToQuery(sql, 1); + LOGGER.debug("Query to execute is [{}]", metadataQuery); + + conn = dbcpDataSource.getConnection(); + ps = conn.prepareStatement(metadataQuery); + rs = ps.executeQuery(); + + ResultSetMetaData metadata = rs.getMetaData(); + int numColumns = metadata.getColumnCount(); + List columnNames = new ArrayList(numColumns); + for (int i = 0; i < numColumns; i++) { + columnNames.add(metadata.getColumnName(i + 1)); + } + + return columnNames; + } + catch (Exception e) { + LOGGER.error("Error while trying to get column names.", e); + throw new HiveJdbcDatabaseAccessException("Error while trying to get column names: " + e.getMessage(), e); + } + finally { + cleanupResources(conn, ps, rs); + } + + } + + + @Override + public int getTotalNumberOfRecords(Configuration conf) throws HiveJdbcDatabaseAccessException { + Connection conn = null; + PreparedStatement ps = null; + ResultSet rs = null; + + try { + initializeDatabaseConnection(conf); + String sql = JdbcStorageConfigManager.getQueryToExecute(conf); + String countQuery = "SELECT COUNT(*) FROM (" + sql + ") tmptable"; + LOGGER.debug("Query to execute is [{}]", countQuery); + + conn = dbcpDataSource.getConnection(); + ps = conn.prepareStatement(countQuery); + rs = ps.executeQuery(); + if (rs.next()) { + return rs.getInt(1); + } + else { + LOGGER.warn("The count query did not return any results.", countQuery); + throw new HiveJdbcDatabaseAccessException("Count query did not return any results."); + } + } + catch (HiveJdbcDatabaseAccessException he) { + throw he; + } + catch (Exception e) { + LOGGER.error("Caught exception while trying to get the number of records", e); + throw new HiveJdbcDatabaseAccessException(e); + } + finally { + cleanupResources(conn, ps, rs); + } + } + + + @Override + public JdbcRecordIterator + getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException { + + Connection conn = null; + PreparedStatement ps = null; + ResultSet rs = null; + + try { + initializeDatabaseConnection(conf); + String sql = JdbcStorageConfigManager.getQueryToExecute(conf); + String limitQuery = addLimitAndOffsetToQuery(sql, limit, offset); + LOGGER.debug("Query to execute is [{}]", limitQuery); + + conn = dbcpDataSource.getConnection(); + ps = conn.prepareStatement(limitQuery, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ps.setFetchSize(getFetchSize(conf)); + rs = ps.executeQuery(); + + return new JdbcRecordIterator(conn, ps, rs); + } + catch (Exception e) { + LOGGER.error("Caught exception while trying to execute query", e); + cleanupResources(conn, ps, rs); + throw new HiveJdbcDatabaseAccessException("Caught exception while trying to execute query", e); + } + } + + + /** + * Uses generic JDBC escape functions to add a limit and offset clause to a query string + * + * @param sql + * @param limit + * @param offset + * @return + */ + protected String addLimitAndOffsetToQuery(String sql, int limit, int offset) { + if (offset == 0) { + return addLimitToQuery(sql, limit); + } + else { + return sql + " {LIMIT " + limit + " OFFSET " + offset + "}"; + } + } + + + /* + * Uses generic JDBC escape functions to add a limit clause to a query string + */ + protected String addLimitToQuery(String sql, int limit) { + return sql + " {LIMIT " + limit + "}"; + } + + + protected void cleanupResources(Connection conn, PreparedStatement ps, ResultSet rs) { + try { + if (rs != null) { + rs.close(); + } + } catch (SQLException e) { + LOGGER.warn("Caught exception during resultset cleanup.", e); + } + + try { + if (ps != null) { + ps.close(); + } + } catch (SQLException e) { + LOGGER.warn("Caught exception during statement cleanup.", e); + } + + try { + if (conn != null) { + conn.close(); + } + } catch (SQLException e) { + LOGGER.warn("Caught exception during connection cleanup.", e); + } + } + + protected void initializeDatabaseConnection(Configuration conf) throws Exception { + if (dbcpDataSource == null) { + synchronized (this) { + if (dbcpDataSource == null) { + Properties props = getConnectionPoolProperties(conf); + dbcpDataSource = BasicDataSourceFactory.createDataSource(props); + } + } + } + } + + + protected Properties getConnectionPoolProperties(Configuration conf) { + // Create the default properties object + Properties dbProperties = getDefaultDBCPProperties(); + + // override with user defined properties + Map userProperties = conf.getValByRegex(DBCP_CONFIG_PREFIX + "\\.*"); + if ((userProperties != null) && (!userProperties.isEmpty())) { + for (Entry entry : userProperties.entrySet()) { + dbProperties.put(entry.getKey().replaceFirst(DBCP_CONFIG_PREFIX + "\\.", ""), entry.getValue()); + } + } + + // essential properties that shouldn't be overridden by users + dbProperties.put("url", conf.get(JdbcStorageConfig.JDBC_URL.getPropertyName())); + dbProperties.put("driverClassName", conf.get(JdbcStorageConfig.JDBC_DRIVER_CLASS.getPropertyName())); + dbProperties.put("type", "javax.sql.DataSource"); + return dbProperties; + } + + + protected Properties getDefaultDBCPProperties() { + Properties props = new Properties(); + props.put("initialSize", "1"); + props.put("maxActive", "3"); + props.put("maxIdle", "0"); + props.put("maxWait", "10000"); + props.put("timeBetweenEvictionRunsMillis", "30000"); + return props; + } + + + protected int getFetchSize(Configuration conf) { + return conf.getInt(JdbcStorageConfig.JDBC_FETCH_SIZE.getPropertyName(), DEFAULT_FETCH_SIZE); + } +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java new file mode 100644 index 0000000..4262502 --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java @@ -0,0 +1,104 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.dao; + +import org.apache.hadoop.io.NullWritable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * An iterator that allows iterating through a SQL resultset. Includes methods to clear up resources. + */ +public class JdbcRecordIterator implements Iterator> { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcRecordIterator.class); + + private Connection conn; + private PreparedStatement ps; + private ResultSet rs; + + + public JdbcRecordIterator(Connection conn, PreparedStatement ps, ResultSet rs) { + this.conn = conn; + this.ps = ps; + this.rs = rs; + } + + + @Override + public boolean hasNext() { + try { + return rs.next(); + } + catch (Exception se) { + LOGGER.warn("hasNext() threw exception", se); + return false; + } + } + + + @Override + public Map next() { + try { + ResultSetMetaData metadata = rs.getMetaData(); + int numColumns = metadata.getColumnCount(); + Map record = new HashMap(numColumns); + for (int i = 0; i < numColumns; i++) { + String key = metadata.getColumnName(i + 1); + String value = rs.getString(i + 1); + if (value == null) { + value = NullWritable.get().toString(); + } + record.put(key, value); + } + + return record; + } + catch (Exception e) { + LOGGER.warn("next() threw exception", e); + return null; + } + } + + + @Override + public void remove() { + throw new UnsupportedOperationException("Remove is not supported"); + } + + + /** + * Release all DB resources + */ + public void close() { + try { + rs.close(); + ps.close(); + conn.close(); + } + catch (Exception e) { + LOGGER.warn("Caught exception while trying to close database objects", e); + } + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/MySqlDatabaseAccessor.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/MySqlDatabaseAccessor.java new file mode 100644 index 0000000..7d821d8 --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/MySqlDatabaseAccessor.java @@ -0,0 +1,39 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.dao; + +/** + * MySQL specific data accessor. This is needed because MySQL JDBC drivers do not support generic LIMIT and OFFSET + * escape functions + */ +public class MySqlDatabaseAccessor extends GenericJdbcDatabaseAccessor { + + @Override + protected String addLimitAndOffsetToQuery(String sql, int limit, int offset) { + if (offset == 0) { + return addLimitToQuery(sql, limit); + } + else { + return sql + " LIMIT " + limit + "," + offset; + } + } + + + @Override + protected String addLimitToQuery(String sql, int limit) { + return sql + " LIMIT " + limit; + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/exception/HiveJdbcDatabaseAccessException.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/exception/HiveJdbcDatabaseAccessException.java new file mode 100644 index 0000000..cde859f --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/exception/HiveJdbcDatabaseAccessException.java @@ -0,0 +1,41 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.exception; + +public class HiveJdbcDatabaseAccessException extends HiveJdbcStorageException { + + private static final long serialVersionUID = -4106595742876276803L; + + + public HiveJdbcDatabaseAccessException() { + super(); + } + + + public HiveJdbcDatabaseAccessException(String message, Throwable cause) { + super(message, cause); + } + + + public HiveJdbcDatabaseAccessException(String message) { + super(message); + } + + + public HiveJdbcDatabaseAccessException(Throwable cause) { + super(cause); + } + +} diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/exception/HiveJdbcStorageException.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/exception/HiveJdbcStorageException.java new file mode 100644 index 0000000..1317838 --- /dev/null +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/exception/HiveJdbcStorageException.java @@ -0,0 +1,40 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.exception; + +public class HiveJdbcStorageException extends Exception { + + private static final long serialVersionUID = 4858210651037826401L; + + + public HiveJdbcStorageException() { + super(); + } + + + public HiveJdbcStorageException(String message) { + super(message); + } + + + public HiveJdbcStorageException(Throwable cause) { + super(cause); + } + + + public HiveJdbcStorageException(String message, Throwable cause) { + super(message, cause); + } +} diff --git jdbc-handler/src/test/java/org/apache/TestSuite.java jdbc-handler/src/test/java/org/apache/TestSuite.java new file mode 100644 index 0000000..df8eab7 --- /dev/null +++ jdbc-handler/src/test/java/org/apache/TestSuite.java @@ -0,0 +1,29 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; + +import org.apache.hive.config.JdbcStorageConfigManagerTest; +import org.apache.hive.storage.jdbc.QueryConditionBuilderTest; +import org.apache.hive.storage.jdbc.dao.GenericJdbcDatabaseAccessorTest; + +@RunWith(Suite.class) +@SuiteClasses({ JdbcStorageConfigManagerTest.class, GenericJdbcDatabaseAccessorTest.class, + QueryConditionBuilderTest.class }) +public class TestSuite { +} diff --git jdbc-handler/src/test/java/org/apache/hive/config/JdbcStorageConfigManagerTest.java jdbc-handler/src/test/java/org/apache/hive/config/JdbcStorageConfigManagerTest.java new file mode 100644 index 0000000..c950831 --- /dev/null +++ jdbc-handler/src/test/java/org/apache/hive/config/JdbcStorageConfigManagerTest.java @@ -0,0 +1,87 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.config; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + +import org.junit.Test; + +import org.apache.hive.storage.jdbc.conf.DatabaseType; +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfigManager; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public class JdbcStorageConfigManagerTest { + + @Test + public void testWithAllRequiredSettingsDefined() { + Properties props = new Properties(); + props.put(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), DatabaseType.MYSQL.toString()); + props.put(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc://localhost:3306/hive"); + props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); + props.put(JdbcStorageConfig.JDBC_DRIVER_CLASS.getPropertyName(), "com.mysql.jdbc.Driver"); + + Map jobMap = new HashMap(); + JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); + + assertThat(jobMap, is(notNullValue())); + assertThat(jobMap.size(), is(equalTo(4))); + assertThat(jobMap.get(JdbcStorageConfig.DATABASE_TYPE.getPropertyName()), is(equalTo("MYSQL"))); + assertThat(jobMap.get(JdbcStorageConfig.JDBC_URL.getPropertyName()), is(equalTo("jdbc://localhost:3306/hive"))); + assertThat(jobMap.get(JdbcStorageConfig.QUERY.getPropertyName()), + is(equalTo("SELECT col1,col2,col3 FROM sometable"))); + } + + + @Test(expected = IllegalArgumentException.class) + public void testWithJdbcUrlMissing() { + Properties props = new Properties(); + props.put(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), DatabaseType.MYSQL.toString()); + props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); + + Map jobMap = new HashMap(); + JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); + } + + + @Test(expected = IllegalArgumentException.class) + public void testWithDatabaseTypeMissing() { + Properties props = new Properties(); + props.put(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc://localhost:3306/hive"); + props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); + + Map jobMap = new HashMap(); + JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); + } + + + @Test(expected = IllegalArgumentException.class) + public void testWithUnknownDatabaseType() { + Properties props = new Properties(); + props.put(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), "Postgres"); + props.put(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc://localhost:3306/hive"); + props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); + + Map jobMap = new HashMap(); + JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); + } + +} diff --git jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/JdbcInputFormatTest.java jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/JdbcInputFormatTest.java new file mode 100644 index 0000000..e66a4e6 --- /dev/null +++ jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/JdbcInputFormatTest.java @@ -0,0 +1,71 @@ +/* + Copyright © 2014 QuBitDigital.com + All rights reserved. +*/ +package org.apache.hive.storage.jdbc; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import org.apache.hive.storage.jdbc.dao.DatabaseAccessor; +import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException; + +import java.io.IOException; + +@RunWith(MockitoJUnitRunner.class) +public class JdbcInputFormatTest { + + @Mock + private DatabaseAccessor mockDatabaseAccessor; + + + @Test + public void testSplitLogic_noSpillOver() throws HiveJdbcDatabaseAccessException, IOException { + JdbcInputFormat f = new JdbcInputFormat(); + when(mockDatabaseAccessor.getTotalNumberOfRecords(any(Configuration.class))).thenReturn(15); + f.setDbAccessor(mockDatabaseAccessor); + + JobConf conf = new JobConf(); + conf.set("mapred.input.dir", "/temp"); + InputSplit[] splits = f.getSplits(conf, 3); + + assertThat(splits, is(notNullValue())); + assertThat(splits.length, is(3)); + + assertThat(splits[0].getLength(), is(5L)); + } + + + @Test + public void testSplitLogic_withSpillOver() throws HiveJdbcDatabaseAccessException, IOException { + JdbcInputFormat f = new JdbcInputFormat(); + when(mockDatabaseAccessor.getTotalNumberOfRecords(any(Configuration.class))).thenReturn(15); + f.setDbAccessor(mockDatabaseAccessor); + + JobConf conf = new JobConf(); + conf.set("mapred.input.dir", "/temp"); + InputSplit[] splits = f.getSplits(conf, 6); + + assertThat(splits, is(notNullValue())); + assertThat(splits.length, is(6)); + + for (int i = 0; i < 3; i++) { + assertThat(splits[i].getLength(), is(3L)); + } + + for (int i = 3; i < 6; i++) { + assertThat(splits[i].getLength(), is(2L)); + } + } +} diff --git jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/QueryConditionBuilderTest.java jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/QueryConditionBuilderTest.java new file mode 100644 index 0000000..5cdae47 --- /dev/null +++ jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/QueryConditionBuilderTest.java @@ -0,0 +1,151 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import static org.hamcrest.Matchers.equalToIgnoringWhiteSpace; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.plan.TableScanDesc; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; + +import java.io.IOException; +import java.util.Scanner; + +public class QueryConditionBuilderTest { + + private static String condition1; + private static String condition2; + + + @BeforeClass + public static void setup() throws IOException { + condition1 = readFileContents("condition1.xml"); + condition2 = readFileContents("condition2.xml"); + } + + + private static String readFileContents(String name) throws IOException { + try (Scanner s = new Scanner(QueryConditionBuilderTest.class.getClassLoader().getResourceAsStream(name))) { + return s.useDelimiter("\\Z").next(); + } + } + + + @Test + public void testSimpleCondition_noTranslation() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("(visitor_id = 'x')"))); + } + + + @Test + public void testSimpleCondition_withTranslation() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), + "visitor_id=vid, sentiment=sentiment, tracking_id=tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("(vid = 'x')"))); + } + + + @Test + public void testSimpleCondition_withDateType() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), + "visitor_id=vid:date, sentiment=sentiment, tracking_id=tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')"))); + } + + + @Test + public void testSimpleCondition_withVariedCaseMappings() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_ID,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), + "visitor_id=VID:date, sentiment=sentiment, tracking_id=tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')"))); + } + + + @Test + public void testMultipleConditions_noTranslation() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("((visitor_id = 'x') and (sentiment = 'y'))"))); + } + + + @Test + public void testMultipleConditions_withTranslation() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("((v = 'x') and (s = 'y'))"))); + } + + + @Test + public void testWithNullConf() { + String condition = QueryConditionBuilder.getInstance().buildCondition(null); + assertThat(condition, is(notNullValue())); + assertThat(condition.trim().isEmpty(), is(true)); + } + + + @Test + public void testWithUndefinedFilterExpr() { + Configuration conf = new Configuration(); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition.trim().isEmpty(), is(true)); + } + +} diff --git jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessorTest.java jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessorTest.java new file mode 100644 index 0000000..5fd600b --- /dev/null +++ jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessorTest.java @@ -0,0 +1,206 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.dao; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.equalToIgnoringCase; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; + +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; +import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException; + +import java.util.List; +import java.util.Map; + +public class GenericJdbcDatabaseAccessorTest { + + @Test + public void testGetColumnNames_starQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + List columnNames = accessor.getColumnNames(conf); + + assertThat(columnNames, is(notNullValue())); + assertThat(columnNames.size(), is(equalTo(7))); + assertThat(columnNames.get(0), is(equalToIgnoringCase("strategy_id"))); + } + + + @Test + public void testGetColumnNames_fieldListQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select name,referrer from test_strategy"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + List columnNames = accessor.getColumnNames(conf); + + assertThat(columnNames, is(notNullValue())); + assertThat(columnNames.size(), is(equalTo(2))); + assertThat(columnNames.get(0), is(equalToIgnoringCase("name"))); + } + + + @Test(expected = HiveJdbcDatabaseAccessException.class) + public void testGetColumnNames_invalidQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from invalid_strategy"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + @SuppressWarnings("unused") + List columnNames = accessor.getColumnNames(conf); + } + + + @Test + public void testGetTotalNumberOfRecords() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + int numRecords = accessor.getTotalNumberOfRecords(conf); + + assertThat(numRecords, is(equalTo(5))); + } + + + @Test + public void testGetTotalNumberOfRecords_whereClause() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy where strategy_id = '5'"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + int numRecords = accessor.getTotalNumberOfRecords(conf); + + assertThat(numRecords, is(equalTo(1))); + } + + + @Test + public void testGetTotalNumberOfRecords_noRecords() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy where strategy_id = '25'"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + int numRecords = accessor.getTotalNumberOfRecords(conf); + + assertThat(numRecords, is(equalTo(0))); + } + + + @Test(expected = HiveJdbcDatabaseAccessException.class) + public void testGetTotalNumberOfRecords_invalidQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from strategyx where strategy_id = '5'"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + @SuppressWarnings("unused") + int numRecords = accessor.getTotalNumberOfRecords(conf); + } + + + @Test + public void testGetRecordIterator() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 2, 0); + + assertThat(iterator, is(notNullValue())); + + int count = 0; + while (iterator.hasNext()) { + Map record = iterator.next(); + count++; + + assertThat(record, is(notNullValue())); + assertThat(record.size(), is(equalTo(7))); + assertThat(record.get("STRATEGY_ID"), is(equalTo(String.valueOf(count)))); + } + + assertThat(count, is(equalTo(2))); + iterator.close(); + } + + + @Test + public void testGetRecordIterator_offsets() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 2, 2); + + assertThat(iterator, is(notNullValue())); + + int count = 0; + while (iterator.hasNext()) { + Map record = iterator.next(); + count++; + + assertThat(record, is(notNullValue())); + assertThat(record.size(), is(equalTo(7))); + assertThat(record.get("STRATEGY_ID"), is(equalTo(String.valueOf(count + 2)))); + } + + assertThat(count, is(equalTo(2))); + iterator.close(); + } + + + @Test + public void testGetRecordIterator_emptyResultSet() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy where strategy_id = '25'"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 0, 2); + + assertThat(iterator, is(notNullValue())); + assertThat(iterator.hasNext(), is(false)); + iterator.close(); + } + + + @Test + public void testGetRecordIterator_largeOffset() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 10, 25); + + assertThat(iterator, is(notNullValue())); + assertThat(iterator.hasNext(), is(false)); + iterator.close(); + } + + + @Test(expected = HiveJdbcDatabaseAccessException.class) + public void testGetRecordIterator_invalidQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from strategyx"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + @SuppressWarnings("unused") + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 0, 2); + } + + + private Configuration buildConfiguration() { + String scriptPath = + GenericJdbcDatabaseAccessorTest.class.getClassLoader().getResource("test_script.sql") + .getPath(); + Configuration config = new Configuration(); + config.set(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), "H2"); + config.set(JdbcStorageConfig.JDBC_DRIVER_CLASS.getPropertyName(), "org.h2.Driver"); + config.set(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc:h2:mem:test;MODE=MySQL;INIT=runscript from '" + + scriptPath + "'"); + config.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy"); + + return config; + } + +} diff --git jdbc-handler/src/test/resources/condition1.xml jdbc-handler/src/test/resources/condition1.xml new file mode 100644 index 0000000..005fc25 --- /dev/null +++ jdbc-handler/src/test/resources/condition1.xml @@ -0,0 +1,48 @@ + + + + + + + + + visitor_id + + + mysql_test + + + + + string + + + + + + + + + + + + x + + + + + + + + + + + + boolean + + + + + diff --git jdbc-handler/src/test/resources/condition2.xml jdbc-handler/src/test/resources/condition2.xml new file mode 100644 index 0000000..f879297 --- /dev/null +++ jdbc-handler/src/test/resources/condition2.xml @@ -0,0 +1,101 @@ + + + + + + + + + + + + + visitor_id + + + mysql_test + + + + + string + + + + + + + + + + + + x + + + + + + + + + + + + boolean + + + + + + + + + + + + + sentiment + + + mysql_test + + + + + + + + + + + + + y + + + + + + + + + + + + + + + + + + + + + + + diff --git jdbc-handler/src/test/resources/test_script.sql jdbc-handler/src/test/resources/test_script.sql new file mode 100644 index 0000000..5d7f08a --- /dev/null +++ jdbc-handler/src/test/resources/test_script.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS test_strategy; + +CREATE TABLE IF NOT EXISTS test_strategy ( + strategy_id int(11) NOT NULL, + name varchar(50) NOT NULL, + referrer varchar(1024) DEFAULT NULL, + landing varchar(1024) DEFAULT NULL, + priority int(11) DEFAULT NULL, + implementation varchar(512) DEFAULT NULL, + last_modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (strategy_id) +); + + +INSERT INTO test_strategy (strategy_id, name, referrer, landing, priority, implementation, last_modified) VALUES (1,'S1','aaa','abc',1000,NULL,'2012-05-08 15:01:15'); +INSERT INTO test_strategy (strategy_id, name, referrer, landing, priority, implementation, last_modified) VALUES (2,'S2','bbb','def',990,NULL,'2012-05-08 15:01:15'); +INSERT INTO test_strategy (strategy_id, name, referrer, landing, priority, implementation, last_modified) VALUES (3,'S3','ccc','ghi',1000,NULL,'2012-05-08 15:01:15'); +INSERT INTO test_strategy (strategy_id, name, referrer, landing, priority, implementation, last_modified) VALUES (4,'S4','ddd','jkl',980,NULL,'2012-05-08 15:01:15'); +INSERT INTO test_strategy (strategy_id, name, referrer, landing, priority, implementation, last_modified) VALUES (5,'S5','eee',NULL,NULL,NULL,'2012-05-08 15:01:15'); + + diff --git packaging/src/main/assembly/src.xml packaging/src/main/assembly/src.xml index e6af8b1..0529e90 100644 --- packaging/src/main/assembly/src.xml +++ packaging/src/main/assembly/src.xml @@ -69,6 +69,7 @@ dev-support/**/* docs/**/* druid-handler/**/* + jdbc-handler/**/* find-bugs/**/* hbase-handler/**/* hcatalog/**/* diff --git pom.xml pom.xml index 3ddec7a..bfa66a1 100644 --- pom.xml +++ pom.xml @@ -39,6 +39,7 @@ contrib druid-handler hbase-handler + jdbc-handler hcatalog hplsql jdbc @@ -137,8 +138,10 @@ 0.9.2 14.0.1 2.4.4 + 1.3.166 2.7.2 ${basedir}/${hive.path.to.root}/testutils/hadoop + 1.1 1.1.1 3.3.0 diff --git ql/src/test/queries/clientpositive/jdbc_handler.q ql/src/test/queries/clientpositive/jdbc_handler.q new file mode 100644 index 0000000..2038617 --- /dev/null +++ ql/src/test/queries/clientpositive/jdbc_handler.q @@ -0,0 +1,58 @@ +CREATE EXTERNAL TABLE tables +( +id int, +db_id int, +name STRING, +type STRING, +owner STRING +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "DERBY", +"hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true", +"hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +"hive.sql.query" = "SELECT TBL_ID, DB_ID, TBL_NAME, TBL_TYPE, OWNER FROM TBLS", +"hive.sql.column.mapping" = "id=TBL_ID, db_id=DB_ID, name=TBL_NAME, type=TBL_TYPE, owner=OWNER", +"hive.sql.dbcp.maxActive" = "1" +); + +CREATE EXTERNAL TABLE dbs +( +id int, +name STRING +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "DERBY", +"hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true", +"hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +"hive.sql.query" = "SELECT DB_ID, NAME FROM DBS", +"hive.sql.column.mapping" = "id=DB_ID, name=NAME", +"hive.sql.dbcp.maxActive" = "1" +); + +select tables.name as tn, dbs.name as dn, tables.type as t +from tables join dbs on (tables.db_id = dbs.id) order by tn, dn, t; + +explain +select + t1.name as a, t2.key as b +from + (select 1 as db_id, tables.name from tables) t1 + join + (select distinct key from src) t2 + on (t2.key-1) = t1.db_id +order by a,b; + +select + t1.name as a, t2.key as b +from + (select 1 as db_id, tables.name from tables) t1 + join + (select distinct key from src) t2 + on (t2.key-1) = t1.db_id +order by a,b; + +show tables; + +describe tables; diff --git ql/src/test/results/clientpositive/llap/jdbc_handler.q.out ql/src/test/results/clientpositive/llap/jdbc_handler.q.out new file mode 100644 index 0000000..74bd60b --- /dev/null +++ ql/src/test/results/clientpositive/llap/jdbc_handler.q.out @@ -0,0 +1,303 @@ +PREHOOK: query: CREATE EXTERNAL TABLE tables +( +id int, +db_id int, +name STRING, +type STRING, +#### A masked pattern was here #### +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "DERBY", +"hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true", +"hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +"hive.sql.query" = "SELECT TBL_ID, DB_ID, TBL_NAME, TBL_TYPE, OWNER FROM TBLS", +#### A masked pattern was here #### +"hive.sql.dbcp.maxActive" = "1" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tables +POSTHOOK: query: CREATE EXTERNAL TABLE tables +( +id int, +db_id int, +name STRING, +type STRING, +#### A masked pattern was here #### +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "DERBY", +"hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true", +"hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +"hive.sql.query" = "SELECT TBL_ID, DB_ID, TBL_NAME, TBL_TYPE, OWNER FROM TBLS", +#### A masked pattern was here #### +"hive.sql.dbcp.maxActive" = "1" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tables +PREHOOK: query: CREATE EXTERNAL TABLE dbs +( +id int, +name STRING +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "DERBY", +"hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true", +"hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +"hive.sql.query" = "SELECT DB_ID, NAME FROM DBS", +"hive.sql.column.mapping" = "id=DB_ID, name=NAME", +"hive.sql.dbcp.maxActive" = "1" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dbs +POSTHOOK: query: CREATE EXTERNAL TABLE dbs +( +id int, +name STRING +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "DERBY", +"hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true", +"hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +"hive.sql.query" = "SELECT DB_ID, NAME FROM DBS", +"hive.sql.column.mapping" = "id=DB_ID, name=NAME", +"hive.sql.dbcp.maxActive" = "1" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dbs +PREHOOK: query: select tables.name as tn, dbs.name as dn, tables.type as t +from tables join dbs on (tables.db_id = dbs.id) order by tn, dn, t +PREHOOK: type: QUERY +PREHOOK: Input: default@dbs +PREHOOK: Input: default@tables +#### A masked pattern was here #### +POSTHOOK: query: select tables.name as tn, dbs.name as dn, tables.type as t +from tables join dbs on (tables.db_id = dbs.id) order by tn, dn, t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dbs +POSTHOOK: Input: default@tables +#### A masked pattern was here #### +alltypesorc default MANAGED_TABLE +cbo_t1 default MANAGED_TABLE +cbo_t2 default MANAGED_TABLE +cbo_t3 default MANAGED_TABLE +dbs default EXTERNAL_TABLE +lineitem default MANAGED_TABLE +part default MANAGED_TABLE +src default MANAGED_TABLE +src1 default MANAGED_TABLE +src_cbo default MANAGED_TABLE +src_json default MANAGED_TABLE +src_sequencefile default MANAGED_TABLE +src_thrift default MANAGED_TABLE +srcbucket default MANAGED_TABLE +srcbucket2 default MANAGED_TABLE +srcpart default MANAGED_TABLE +tables default EXTERNAL_TABLE +PREHOOK: query: explain +select + t1.name as a, t2.key as b +from + (select 1 as db_id, tables.name from tables) t1 + join + (select distinct key from src) t2 + on (t2.key-1) = t1.db_id +order by a,b +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + t1.name as a, t2.key as b +from + (select 1 as db_id, tables.name from tables) t1 + join + (select distinct key from src) t2 + on (t2.key-1) = t1.db_id +order by a,b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: tables + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: name (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: 1.0 (type: double) + sort order: + + Map-reduce partition columns: 1.0 (type: double) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key - 1) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 1.0 (type: double) + 1 (_col0 - 1) (type: double) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 225 Data size: 19618 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 225 Data size: 19618 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 225 Data size: 19618 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 225 Data size: 19618 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 225 Data size: 19618 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col0 - 1) (type: double) + sort order: + + Map-reduce partition columns: (_col0 - 1) (type: double) + Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select + t1.name as a, t2.key as b +from + (select 1 as db_id, tables.name from tables) t1 + join + (select distinct key from src) t2 + on (t2.key-1) = t1.db_id +order by a,b +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@tables +#### A masked pattern was here #### +POSTHOOK: query: select + t1.name as a, t2.key as b +from + (select 1 as db_id, tables.name from tables) t1 + join + (select distinct key from src) t2 + on (t2.key-1) = t1.db_id +order by a,b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@tables +#### A masked pattern was here #### +alltypesorc 2 +cbo_t1 2 +cbo_t2 2 +cbo_t3 2 +dbs 2 +lineitem 2 +part 2 +src 2 +src1 2 +src_cbo 2 +src_json 2 +src_sequencefile 2 +src_thrift 2 +srcbucket 2 +srcbucket2 2 +srcpart 2 +tables 2 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +dbs +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +tables +PREHOOK: query: describe tables +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tables +POSTHOOK: query: describe tables +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tables +id string from deserializer +db_id string from deserializer +name string from deserializer +type string from deserializer +#### A masked pattern was here ####