Index: ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java (revision 0) +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java (revision 0) @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.IOException; +import java.net.ServerSocket; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.util.StringUtils; + +/** + * + * TestHiveRemote. + * + * Tests using the Hive metadata class to make calls to a remote metastore + */ +public class TestHiveRemote extends TestHive { + + /** + * Starts a remote metastore + */ + private static class RunMS implements Runnable { + String port; + + public RunMS(String port) { + this.port = port; + } + + @Override + public void run() { + try { + HiveMetaStore.main(new String[] { port }); + } catch (Throwable e) { + e.printStackTrace(System.err); + assert false; + } + } + + } + + /** + * Start a remote metastore and initialize a Hive object pointing at it. + */ + @Override + protected void setUp() throws Exception { + super.setUp(); + hiveConf = new HiveConf(this.getClass()); + String port = findFreePort(); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); + + Thread t = new Thread(new RunMS(port)); + t.start(); + + // Wait a little bit for the metastore to start. + Thread.sleep(5000); + + + try { + hm = Hive.get(hiveConf); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err + .println("Unable to initialize Hive Metastore using configuration: \n " + + hiveConf); + throw e; + } + } + + /** + * Finds a free port. + * + * @return a free port + * @throws IOException + */ + private String findFreePort() throws IOException { + ServerSocket socket= new ServerSocket(0); + int port = socket.getLocalPort(); + socket.close(); + return String.valueOf(port); + } +} Index: ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (revision 1410521) +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (working copy) @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import junit.framework.TestCase; @@ -33,6 +34,10 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.ql.index.HiveIndex; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; @@ -49,8 +54,8 @@ * */ public class TestHive extends TestCase { - private Hive hm; - private HiveConf hiveConf; + protected Hive hm; + protected HiveConf hiveConf; @Override protected void setUp() throws Exception { @@ -61,7 +66,7 @@ } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); System.err - .println("Unable to initialize Hive Metastore using configruation: \n " + .println("Unable to initialize Hive Metastore using configuration: \n " + hiveConf); throw e; } @@ -145,33 +150,7 @@ } // get table - Table ft = null; - Warehouse wh = new Warehouse(hiveConf); - try { - ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); - ft.checkValidity(); - assertEquals("Table names didn't match for table: " + tableName, tbl - .getTableName(), ft.getTableName()); - assertEquals("Table owners didn't match for table: " + tableName, tbl - .getOwner(), ft.getOwner()); - assertEquals("Table retention didn't match for table: " + tableName, - tbl.getRetention(), ft.getRetention()); - assertEquals("Data location is not set correctly", - wh.getTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(), - ft.getDataLocation().toString()); - // now that URI is set correctly, set the original table's uri and then - // compare the two tables - tbl.setDataLocation(ft.getDataLocation()); - assertTrue("Tables doesn't match: " + tableName, ft.getTTable().equals( - tbl.getTTable())); - assertEquals("Serde is not set correctly", tbl.getDeserializer() - .getClass().getName(), ft.getDeserializer().getClass().getName()); - assertEquals("SerializationLib is not set correctly", tbl - .getSerializationLib(), LazySimpleSerDe.class.getName()); - } catch (HiveException e) { - e.printStackTrace(); - assertTrue("Unable to fetch table correctly: " + tableName, false); - } + validateTable(tbl, tableName); try { hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, true, @@ -218,34 +197,7 @@ assertTrue("Unable to create table: " + tableName, false); } // get table - Warehouse wh = new Warehouse(hiveConf); - Table ft = null; - try { - ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); - assertNotNull("Unable to fetch table", ft); - ft.checkValidity(); - assertEquals("Table names didn't match for table: " + tableName, tbl - .getTableName(), ft.getTableName()); - assertEquals("Table owners didn't match for table: " + tableName, tbl - .getOwner(), ft.getOwner()); - assertEquals("Table retention didn't match for table: " + tableName, - tbl.getRetention(), ft.getRetention()); - assertEquals("Data location is not set correctly", - wh.getTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(), - ft.getDataLocation().toString()); - // now that URI is set correctly, set the original table's uri and then - // compare the two tables - tbl.setDataLocation(ft.getDataLocation()); - assertTrue("Tables doesn't match: " + tableName, ft.getTTable() - .equals(tbl.getTTable())); - assertEquals("SerializationLib is not set correctly", tbl - .getSerializationLib(), ThriftDeserializer.class.getName()); - assertEquals("Serde is not set correctly", tbl.getDeserializer() - .getClass().getName(), ft.getDeserializer().getClass().getName()); - } catch (HiveException e) { - System.err.println(StringUtils.stringifyException(e)); - assertTrue("Unable to fetch table correctly: " + tableName, false); - } + validateTable(tbl, tableName); hm.dropTable(DEFAULT_DATABASE_NAME, tableName); } catch (Throwable e) { System.err.println(StringUtils.stringifyException(e)); @@ -254,6 +206,47 @@ } } + /** + * Gets a table from the metastore and compares it to the original Table + * + * @param tbl + * @param tableName + * @throws MetaException + */ + private void validateTable(Table tbl, String tableName) throws MetaException { + Warehouse wh = new Warehouse(hiveConf); + Table ft = null; + try { + ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + assertNotNull("Unable to fetch table", ft); + ft.checkValidity(); + assertEquals("Table names didn't match for table: " + tableName, tbl + .getTableName(), ft.getTableName()); + assertEquals("Table owners didn't match for table: " + tableName, tbl + .getOwner(), ft.getOwner()); + assertEquals("Table retention didn't match for table: " + tableName, + tbl.getRetention(), ft.getRetention()); + assertEquals("Data location is not set correctly", + wh.getTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(), + ft.getDataLocation().toString()); + // now that URI and times are set correctly, set the original table's uri and times + // and then compare the two tables + tbl.setDataLocation(ft.getDataLocation()); + tbl.setCreateTime(ft.getTTable().getCreateTime()); + tbl.getParameters().put(hive_metastoreConstants.DDL_TIME, + ft.getParameters().get(hive_metastoreConstants.DDL_TIME)); + assertTrue("Tables doesn't match: " + tableName, ft.getTTable() + .equals(tbl.getTTable())); + assertEquals("SerializationLib is not set correctly", tbl + .getSerializationLib(), ft.getSerializationLib()); + assertEquals("Serde is not set correctly", tbl.getDeserializer() + .getClass().getName(), ft.getDeserializer().getClass().getName()); + } catch (HiveException e) { + System.err.println(StringUtils.stringifyException(e)); + assertTrue("Unable to fetch table correctly: " + tableName, false); + } + } + private static Table createTestTable(String dbName, String tableName) throws HiveException { Table tbl = new Table(dbName, tableName); tbl.setInputFormatClass(SequenceFileInputFormat.class.getName()); @@ -375,6 +368,125 @@ } } + /** + * Tests creating a simple index on a simple table. + * + * @throws Throwable + */ + public void testIndex() throws Throwable { + try{ + // create a simple table + String tableName = "table_for_testindex"; + try { + hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + } catch (HiveException e) { + e.printStackTrace(); + assertTrue("Unable to drop table", false); + } + + Table tbl = new Table(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + List fields = tbl.getCols(); + + fields.add(new FieldSchema("col1", serdeConstants.INT_TYPE_NAME, "int -- first column")); + fields.add(new FieldSchema("col2", serdeConstants.STRING_TYPE_NAME, + "string -- second column")); + fields.add(new FieldSchema("col3", serdeConstants.DOUBLE_TYPE_NAME, + "double -- thrift column")); + tbl.setFields(fields); + + tbl.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); + tbl.setInputFormatClass(SequenceFileInputFormat.class); + + // create table + try { + hm.createTable(tbl); + } catch (HiveException e) { + e.printStackTrace(); + assertTrue("Unable to create table: " + tableName, false); + } + + // Create a simple index + String indexName = "index_on_table_for_testindex"; + String indexHandlerClass = HiveIndex.IndexType.COMPACT_SUMMARY_TABLE.getHandlerClsName(); + List indexedCols = new ArrayList(); + indexedCols.add("col1"); + String indexTableName = "index_on_table_for_testindex_table"; + boolean deferredRebuild = true; + String inputFormat = SequenceFileInputFormat.class.getName(); + String outputFormat = SequenceFileOutputFormat.class.getName(); + String serde = null; + String storageHandler = null; + String location = null; + String collItemDelim = null; + String fieldDelim = null; + String fieldEscape = null; + String lineDelim = null; + String mapKeyDelim = null; + String indexComment = null; + Map indexProps = null; + Map tableProps = null; + Map serdeProps = new HashMap(); + hm.createIndex(tableName, indexName, indexHandlerClass, indexedCols, indexTableName, + deferredRebuild, inputFormat, outputFormat, serde, storageHandler, location, + indexProps, tableProps, serdeProps, collItemDelim, fieldDelim, fieldEscape, lineDelim, + mapKeyDelim, indexComment); + + // Retrieve and validate the index + Index index = null; + try { + index = hm.getIndex(tableName, indexName); + assertNotNull("Unable to fetch index", index); + index.validate(); + assertEquals("Index names don't match for index: " + indexName, indexName, + index.getIndexName()); + assertEquals("Table names don't match for index: " + indexName, tableName, + index.getOrigTableName()); + assertEquals("Index table names didn't match for index: " + indexName, indexTableName, + index.getIndexTableName()); + assertEquals("Index handler classes didn't match for index: " + indexName, + indexHandlerClass, index.getIndexHandlerClass()); + assertEquals("Deferred rebuild didn't match for index: " + indexName, deferredRebuild, + index.isDeferredRebuild()); + + } catch (HiveException e) { + System.err.println(StringUtils.stringifyException(e)); + assertTrue("Unable to fetch index correctly: " + indexName, false); + } + + // Drop index + try { + hm.dropIndex(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, indexName, true); + } catch (HiveException e) { + System.err.println(StringUtils.stringifyException(e)); + assertTrue("Unable to drop index: " + indexName, false); + } + + boolean dropIndexException = false; + try { + hm.getIndex(tableName, indexName); + } catch (HiveException e) { + // Expected since it was just dropped + dropIndexException = true; + } + + assertTrue("Unable to drop index: " + indexName, dropIndexException); + + // Drop table + try { + hm.dropTable(tableName); + Table droppedTable = hm.getTable(tableName, false); + assertNull("Unable to drop table " + tableName, droppedTable); + } catch (HiveException e) { + System.err.println(StringUtils.stringifyException(e)); + assertTrue("Unable to drop table: " + tableName, false); + } + } catch (Throwable e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testIndex failed"); + throw e; + } + } + public void testHiveRefreshDatabase() throws Throwable{ String testDatabaseName = "test_database"; Database testDatabase = new Database(); Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 1410521) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy) @@ -56,7 +56,6 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; @@ -72,6 +71,7 @@ import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.index.HiveIndexHandler; import org.apache.hadoop.hive.ql.session.CreateTableAutomaticGrant; @@ -739,7 +739,9 @@ Index indexDesc = new Index(indexName, indexHandlerClass, dbName, tableName, time, time, indexTblName, storageDescriptor, params, deferredRebuild); - indexDesc.getParameters().put("comment", indexComment); + if (indexComment != null) { + indexDesc.getParameters().put("comment", indexComment); + } if (idxProps != null) {