diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestGetListIndexes.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestGetListIndexes.java new file mode 100644 index 0000000..32c40d1 --- /dev/null +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestGetListIndexes.java @@ -0,0 +1,551 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.client; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.IndexBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestGetListIndexes { + // Needed until there is no junit release with @BeforeParam, @AfterParam (junit 4.13) + // https://github.com/junit-team/junit4/commit/1bf8438b65858565dbb64736bfe13aae9cfc1b5a + // Then we should remove our own copy + private static Set metaStoreServices = null; + private AbstractMetaStoreService metaStore; + private IMetaStoreClient client; + + private final static String DB_NAME = "testindexdb"; + private final static String TABLE_NAME = "testindextable"; + private final static String INDEX_NAME = "testcreateindex"; + private final static short MAX = 10; + + @Parameterized.Parameters(name = "{0}") + public static List getMetaStoreToTest() throws Exception { + List result = MetaStoreFactoryForTests.getMetaStores(); + metaStoreServices = result.stream() + .map(test -> (AbstractMetaStoreService)test[1]) + .collect(Collectors.toSet()); + return result; + } + + public TestGetListIndexes(String name, AbstractMetaStoreService metaStore) throws Exception { + this.metaStore = metaStore; + this.metaStore.start(); + } + + // Needed until there is no junit release with @BeforeParam, @AfterParam (junit 4.13) + // https://github.com/junit-team/junit4/commit/1bf8438b65858565dbb64736bfe13aae9cfc1b5a + // Then we should move this to @AfterParam + @AfterClass + public static void stopMetaStores() throws Exception { + for(AbstractMetaStoreService metaStoreService : metaStoreServices) { + metaStoreService.stop(); + } + } + + @Before + public void setUp() throws Exception { + // Get new client + client = metaStore.getClient(); + + // Clean up the database + client.dropDatabase(DB_NAME, true, true, true); + metaStore.cleanWarehouseDirs(); + createDB(DB_NAME); + } + + @After + public void tearDown() throws Exception { + try { + if (client != null) { + client.close(); + } + } finally { + client = null; + } + } + + // Get index tests + + @Test(expected = NoSuchObjectException.class) + public void testGetNonExistingIndex() throws Exception { + + createIndex(INDEX_NAME, TABLE_NAME); + client.getIndex(DB_NAME, TABLE_NAME, "nonexisingindex"); + client.dropIndex(DB_NAME, TABLE_NAME, INDEX_NAME, true); + } + + @Test(expected = NoSuchObjectException.class) + public void testGetIndexNonExistingTable() throws Exception { + + createIndex(INDEX_NAME, TABLE_NAME); + client.getIndex(DB_NAME, "nonexistingtable", INDEX_NAME); + client.dropIndex(DB_NAME, TABLE_NAME, INDEX_NAME, true); + } + + @Test(expected = NoSuchObjectException.class) + public void testGetIndexNonExistingDatabase() throws Exception { + + createIndex(INDEX_NAME, TABLE_NAME); + client.getIndex("nonexistingdb", TABLE_NAME, INDEX_NAME); + client.dropIndex(DB_NAME, TABLE_NAME, INDEX_NAME, true); + } + + @Test(expected = NoSuchObjectException.class) + public void testGetIndexNullName() throws Exception { + + client.getIndex(DB_NAME, TABLE_NAME, null); + } + + @Test(expected = MetaException.class) + public void testGetIndexNullTableName() throws Exception { + + client.getIndex(DB_NAME, null, INDEX_NAME); + } + + @Test(expected = MetaException.class) + public void testGetIndexNullDBName() throws Exception { + + client.getIndex(null, TABLE_NAME, INDEX_NAME); + } + + @Test(expected = NoSuchObjectException.class) + public void testGetIndexEmptyName() throws Exception { + client.getIndex(DB_NAME, TABLE_NAME, ""); + } + + @Test(expected = NoSuchObjectException.class) + public void testGetIndexEmptyTableName() throws Exception { + client.getIndex(DB_NAME, "", INDEX_NAME); + } + + @Test(expected = NoSuchObjectException.class) + public void testGetIndexEmptyDBName() throws Exception { + client.getIndex("", TABLE_NAME, INDEX_NAME); + } + + // List index tests + + @Test + public void testListIndexes() throws Exception { + String origTableName1 = TABLE_NAME + "1"; + String origTableName2 = TABLE_NAME + "2"; + + String indexName1 = "testindexname1"; + String indexName2 = "testindexname2"; + String indexName3 = "testindexname3"; + String indexName4 = "testindexname4"; + + String indexTableName1 = origTableName1 + "__" + indexName1 + "__"; + String indexTableName2 = origTableName1 + "__" + indexName2 + "__"; + String indexTableName3 = origTableName2 + "__" + indexName3 + "__"; + String indexTableName4 = origTableName1 + "__" + indexName4 + "__"; + + String dbName = DB_NAME + "2"; + createDB(dbName); + + Table origTable1 = createTable(DB_NAME, origTableName1); + Table origTable2 = createTable(DB_NAME, origTableName2); + Table origTable3 = createTable(dbName, origTableName1); + createIndex(DB_NAME, origTable1, indexName1, buildTable(DB_NAME, indexTableName1)); + createIndex(DB_NAME, origTable1, indexName2, buildTable(DB_NAME, indexTableName2)); + createIndex(DB_NAME, origTable2, indexName3, buildTable(DB_NAME, indexTableName3)); + createIndex(dbName, origTable3, indexName4, buildTable(dbName, indexTableName4)); + + List indexes = client.listIndexes(DB_NAME, origTableName1, MAX); + Assert.assertNotNull(indexes); + Assert.assertEquals(2, indexes.size()); + Map indexMap = new HashMap<>(); + for (Index index : indexes) { + indexMap.put(index.getIndexName(), index); + } + + Index index = indexMap.get(indexName1); + Assert.assertNotNull(index); + Assert.assertEquals(indexName1, index.getIndexName()); + Assert.assertEquals(origTableName1, index.getOrigTableName()); + Assert.assertEquals(DB_NAME, index.getDbName()); + + index = indexMap.get(indexName2); + Assert.assertNotNull(index); + Assert.assertEquals(indexName2, index.getIndexName()); + Assert.assertEquals(origTableName1, index.getOrigTableName()); + Assert.assertEquals(DB_NAME, index.getDbName()); + + indexes = client.listIndexes(DB_NAME, origTableName2, MAX); + Assert.assertNotNull(indexes); + Assert.assertEquals(1, indexes.size()); + index = indexes.get(0); + Assert.assertNotNull(index); + Assert.assertEquals(indexName3, index.getIndexName()); + Assert.assertEquals(origTableName2, index.getOrigTableName()); + Assert.assertEquals(DB_NAME, index.getDbName()); + + indexes = client.listIndexes(dbName, origTableName1, MAX); + Assert.assertNotNull(indexes); + Assert.assertEquals(1, indexes.size()); + index = indexes.get(0); + Assert.assertNotNull(index); + Assert.assertEquals(indexName4, index.getIndexName()); + Assert.assertEquals(origTableName1, index.getOrigTableName()); + Assert.assertEquals(dbName, index.getDbName()); + + client.dropIndex(DB_NAME, origTableName1, indexName1, true); + client.dropIndex(DB_NAME, origTableName1, indexName2, true); + client.dropIndex(DB_NAME, origTableName2, indexName3, true); + client.dropIndex(dbName, origTableName1, indexName4, true); + client.dropDatabase(dbName, true, true, true); + } + + @Test + public void testListIndexesEmptyList() throws Exception { + + createTable(DB_NAME, TABLE_NAME); + List indexes = client.listIndexes(DB_NAME, TABLE_NAME, MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test + public void testListIndexesInvalidDb() throws Exception { + + List indexes = client.listIndexes("nonexistingdb", TABLE_NAME, MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test + public void testListIndexesInvalidTable() throws Exception { + + List indexes = client.listIndexes(DB_NAME, "nonexsitingtable", MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test(expected = MetaException.class) + public void testListIndexesNullDb() throws Exception { + + client.listIndexes(null, TABLE_NAME, MAX); + } + + @Test(expected = MetaException.class) + public void testListIndexesNullTable() throws Exception { + + client.listIndexes(DB_NAME, null, MAX); + } + + @Test + public void testListIndexesEmptyDb() throws Exception { + + List indexes = client.listIndexes("", TABLE_NAME, MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test + public void testListIndexesEmptyTable() throws Exception { + + List indexes = client.listIndexes(DB_NAME, "", MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test + public void testListIndexesWithDifferentNums() throws Exception { + + // TODO: The max parameter doesn't have any effect + String indexName1 = "testindexname1"; + String indexName2 = "testindexname2"; + String indexName3 = "testindexname3"; + String indexName4 = "testindexname4"; + + String indexTableName1 = TABLE_NAME + "__" + indexName1 + "__"; + String indexTableName2 = TABLE_NAME + "__" + indexName2 + "__"; + String indexTableName3 = TABLE_NAME + "__" + indexName3 + "__"; + String indexTableName4 = TABLE_NAME + "__" + indexName4 + "__"; + + Table origTable = createTable(DB_NAME, TABLE_NAME); + createIndex(DB_NAME, origTable, indexName1, buildTable(DB_NAME, indexTableName1)); + createIndex(DB_NAME, origTable, indexName2, buildTable(DB_NAME, indexTableName2)); + createIndex(DB_NAME, origTable, indexName3, buildTable(DB_NAME, indexTableName3)); + createIndex(DB_NAME, origTable, indexName4, buildTable(DB_NAME, indexTableName4)); + + short num = 2; + List indexes = client.listIndexes(DB_NAME, TABLE_NAME, num); + Assert.assertNotNull(indexes); + Assert.assertEquals(4, indexes.size()); + + num = 1; + indexes = client.listIndexes(DB_NAME, TABLE_NAME, num); + Assert.assertNotNull(indexes); + Assert.assertEquals(4, indexes.size()); + + num = 0; + indexes = client.listIndexes(DB_NAME, TABLE_NAME, num); + Assert.assertNotNull(indexes); + Assert.assertEquals(4, indexes.size()); + + num = -1; + indexes = client.listIndexes(DB_NAME, TABLE_NAME, num); + Assert.assertNotNull(indexes); + Assert.assertEquals(4, indexes.size()); + + client.dropIndex(DB_NAME, TABLE_NAME, indexName1, true); + client.dropIndex(DB_NAME, TABLE_NAME, indexName2, true); + client.dropIndex(DB_NAME, TABLE_NAME, indexName3, true); + client.dropIndex(DB_NAME, TABLE_NAME, indexName4, true); + } + + // List index names tests + + @Test + public void testListIndexNames() throws Exception { + + String origTableName1 = TABLE_NAME + "1"; + String origTableName2 = TABLE_NAME + "2"; + + String indexName1 = "testindexname1"; + String indexName2 = "testindexname2"; + String indexName3 = "testindexname3"; + String indexName4 = "testindexname4"; + + String indexTableName1 = origTableName1 + "__" + indexName1 + "__"; + String indexTableName2 = origTableName1 + "__" + indexName2 + "__"; + String indexTableName3 = origTableName2 + "__" + indexName3 + "__"; + String indexTableName4 = origTableName1 + "__" + indexName4 + "__"; + + String dbName = DB_NAME + "2"; + createDB(dbName); + + Table origTable1 = createTable(DB_NAME, origTableName1); + Table origTable2 = createTable(DB_NAME, origTableName2); + Table origTable3 = createTable(dbName, origTableName1); + createIndex(DB_NAME, origTable1, indexName1, buildTable(DB_NAME, indexTableName1)); + createIndex(DB_NAME, origTable1, indexName2, buildTable(DB_NAME, indexTableName2)); + createIndex(DB_NAME, origTable2, indexName3, buildTable(DB_NAME, indexTableName3)); + createIndex(dbName, origTable3, indexName4, buildTable(dbName, indexTableName4)); + + List indexNames = client.listIndexNames(DB_NAME, origTableName1, MAX); + Assert.assertNotNull(indexNames); + Assert.assertEquals(2, indexNames.size()); + Assert.assertTrue(indexNames.contains(indexName1)); + Assert.assertTrue(indexNames.contains(indexName2)); + + indexNames = client.listIndexNames(DB_NAME, origTableName2, MAX); + Assert.assertNotNull(indexNames); + Assert.assertEquals(1, indexNames.size()); + Assert.assertTrue(indexNames.contains(indexName3)); + + indexNames = client.listIndexNames(dbName, origTableName1, MAX); + Assert.assertNotNull(indexNames); + Assert.assertEquals(1, indexNames.size()); + Assert.assertTrue(indexNames.contains(indexName4)); + + client.dropIndex(DB_NAME, origTableName1, indexName1, true); + client.dropIndex(DB_NAME, origTableName1, indexName2, true); + client.dropIndex(DB_NAME, origTableName2, indexName3, true); + client.dropIndex(dbName, origTableName1, indexName4, true); + client.dropDatabase(dbName, true, true, true); + } + + @Test + public void testListIndexNamesEmptyList() throws Exception { + + createTable(DB_NAME, TABLE_NAME); + List indexes = client.listIndexNames(DB_NAME, TABLE_NAME, MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test + public void testListIndexNamesInvalidDb() throws Exception { + + List indexes = client.listIndexNames("nonexistingdb", TABLE_NAME, MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test + public void testListIndexNamesInvalidTable() throws Exception { + + List indexes = client.listIndexNames(DB_NAME, "nonexsitingtable", MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test(expected = MetaException.class) + public void testListIndexNamesNullDb() throws Exception { + + client.listIndexNames(null, TABLE_NAME, MAX); + } + + @Test(expected = MetaException.class) + public void testListIndexNamesNullTable() throws Exception { + + client.listIndexNames(DB_NAME, null, MAX); + } + + @Test + public void testListIndexNamesEmptyDb() throws Exception { + + List indexes = client.listIndexNames("", TABLE_NAME, MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test + public void testListIndexNamesEmptyTable() throws Exception { + + List indexes = client.listIndexNames(DB_NAME, "", MAX); + Assert.assertNotNull(indexes); + Assert.assertTrue(indexes.isEmpty()); + } + + @Test + public void testListIndexNamesWithDifferentNums() throws Exception { + + // TODO: The max parameter doesn't have any effect + String indexName1 = "testindexname1"; + String indexName2 = "testindexname2"; + String indexName3 = "testindexname3"; + String indexName4 = "testindexname4"; + + String indexTableName1 = TABLE_NAME + "__" + indexName1 + "__"; + String indexTableName2 = TABLE_NAME + "__" + indexName2 + "__"; + String indexTableName3 = TABLE_NAME + "__" + indexName3 + "__"; + String indexTableName4 = TABLE_NAME + "__" + indexName4 + "__"; + + Table origTable = createTable(DB_NAME, TABLE_NAME); + createIndex(DB_NAME, origTable, indexName1, buildTable(DB_NAME, indexTableName1)); + createIndex(DB_NAME, origTable, indexName2, buildTable(DB_NAME, indexTableName2)); + createIndex(DB_NAME, origTable, indexName3, buildTable(DB_NAME, indexTableName3)); + createIndex(DB_NAME, origTable, indexName4, buildTable(DB_NAME, indexTableName4)); + + short num = 2; + List indexes = client.listIndexNames(DB_NAME, TABLE_NAME, num); + Assert.assertNotNull(indexes); + Assert.assertEquals(4, indexes.size()); + + num = 1; + indexes = client.listIndexNames(DB_NAME, TABLE_NAME, num); + Assert.assertNotNull(indexes); + Assert.assertEquals(4, indexes.size()); + + num = 0; + indexes = client.listIndexNames(DB_NAME, TABLE_NAME, num); + Assert.assertNotNull(indexes); + Assert.assertEquals(4, indexes.size()); + + num = -1; + indexes = client.listIndexNames(DB_NAME, TABLE_NAME, num); + Assert.assertNotNull(indexes); + Assert.assertEquals(4, indexes.size()); + + client.dropIndex(DB_NAME, TABLE_NAME, indexName1, true); + client.dropIndex(DB_NAME, TABLE_NAME, indexName2, true); + client.dropIndex(DB_NAME, TABLE_NAME, indexName3, true); + client.dropIndex(DB_NAME, TABLE_NAME, indexName4, true); + } + + // Helper methods + + private Table createTable(String dbName, String tableName) throws Exception { + Table table = buildTable(dbName, tableName); + client.createTable(table); + return table; + } + + private Table buildTable(String dbName, String tableName) throws Exception { + Table table = new TableBuilder() + .setDbName(dbName) + .setTableName(tableName) + .addCol("id", "int", "test col id") + .addCol("value", "string", "test col value") + .addStorageDescriptorParam("testSDParamKey", "testSDParamValue") + .setSerdeName(tableName) + .setStoredAsSubDirectories(false) + .addSerdeParam("testSerdeParamKey", "testSerdeParamValue") + .setLocation(metaStore.getWarehouseRoot() + "/" + tableName) + .build(); + return table; + } + + private void createIndex(String indexName, String origTableName) throws Exception { + Table origTable = createTable(DB_NAME, origTableName); + String indexTableName = origTableName + "__" + indexName + "__"; + createIndex(DB_NAME, origTable, indexName, buildTable(DB_NAME, indexTableName)); + } + + private void createIndex(String dbName, Table origTable, String indexName, Table indexTable) + throws Exception { + int createTime = (int) (System.currentTimeMillis() / 1000); + Map params = new HashMap(); + params.put("indexParamKey", "indexParamValue"); + + Index index = + buildIndex(dbName, origTable.getTableName(), indexName, indexTable.getTableName()); + index.setCreateTime(createTime); + index.setLastAccessTime(createTime); + index.setParameters(params); + client.createIndex(index, indexTable); + } + + private Index buildIndex(String dbName, String origTableName, String indexName, + String indexTableName) throws MetaException { + Index index = new IndexBuilder() + .setDbName(dbName) + .setTableName(origTableName) + .setIndexName(indexName) + .setIndexTableName(indexTableName) + .addCol("id", "int", "test col id") + .addCol("value", "string", "test col value") + .setDeferredRebuild(false) + .build(); + return index; + } + + private void createDB(String dbName) throws TException { + Database db = new DatabaseBuilder(). + setName(dbName). + build(); + client.createDatabase(db); + } +}