diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index 3b5f65f..4dbd26a 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -29,26 +29,33 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; +import org.apache.hadoop.hive.metastore.events.PreAddIndexEvent; import org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.PreAlterIndexEvent; import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; import org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; import org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.PreDropIndexEvent; import org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreDropTableEvent; import org.apache.hadoop.hive.metastore.events.PreEventContext; @@ -183,12 +190,35 @@ private void validateDropDb(Database expectedDb, Database actualDb) { assertEquals(expectedDb, actualDb); } + private void validateIndex(Index expectedIndex, Index actualIndex) { + assertEquals(expectedIndex.getDbName(), actualIndex.getDbName()); + assertEquals(expectedIndex.getIndexName(), actualIndex.getIndexName()); + assertEquals(expectedIndex.getIndexHandlerClass(), actualIndex.getIndexHandlerClass()); + assertEquals(expectedIndex.getOrigTableName(), actualIndex.getOrigTableName()); + assertEquals(expectedIndex.getIndexTableName(), actualIndex.getIndexTableName()); + assertEquals(expectedIndex.getSd().getLocation(), actualIndex.getSd().getLocation()); + } + + private void validateAddIndex(Index expectedIndex, Index actualIndex) { + validateIndex(expectedIndex, actualIndex); + } + + private void validateAlterIndex(Index expectedOldIndex, Index actualOldIndex, + Index expectedNewIndex, Index actualNewIndex) { + validateIndex(expectedOldIndex, actualOldIndex); + validateIndex(expectedNewIndex, actualNewIndex); + } + + private void validateDropIndex(Index expectedIndex, Index actualIndex) { + validateIndex(expectedIndex, actualIndex); + } + public void testListener() throws Exception { int listSize = 0; List notifyList = DummyListener.notifyList; - assertEquals(notifyList.size(), listSize); List preNotifyList = DummyPreListener.notifyList; + assertEquals(notifyList.size(), listSize); assertEquals(preNotifyList.size(), listSize); driver.run("create database " + dbName); @@ -216,6 +246,48 @@ public void testListener() throws Exception { assert tblEvent.getStatus(); validateCreateTable(tbl, tblEvent.getTable()); + driver.run("create index tmptbl_i on table tmptbl(a) as 'compact' " + + "WITH DEFERRED REBUILD IDXPROPERTIES ('prop1'='val1', 'prop2'='val2')"); + listSize += 2; // creates index table internally + assertEquals(notifyList.size(), listSize); + + AddIndexEvent addIndexEvent = (AddIndexEvent)notifyList.get(listSize - 1); + assert addIndexEvent.getStatus(); + PreAddIndexEvent preAddIndexEvent = (PreAddIndexEvent)(preNotifyList.get(preNotifyList.size() - 3)); + + Index oldIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i"); + + validateAddIndex(oldIndex, addIndexEvent.getIndex()); + + validateAddIndex(oldIndex, preAddIndexEvent.getIndex()); + + driver.run("alter index tmptbl_i on tmptbl set IDXPROPERTIES " + + "('prop1'='val1_new', 'prop3'='val3')"); + listSize++; + assertEquals(notifyList.size(), listSize); + + Index newIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i"); + + AlterIndexEvent alterIndexEvent = (AlterIndexEvent) notifyList.get(listSize - 1); + assert alterIndexEvent.getStatus(); + validateAlterIndex(oldIndex, alterIndexEvent.getOldIndex(), + newIndex, alterIndexEvent.getNewIndex()); + + PreAlterIndexEvent preAlterIndexEvent = (PreAlterIndexEvent) (preNotifyList.get(preNotifyList.size() - 1)); + validateAlterIndex(oldIndex, preAlterIndexEvent.getOldIndex(), + newIndex, preAlterIndexEvent.getNewIndex()); + + driver.run("drop index tmptbl_i on tmptbl"); + listSize++; + assertEquals(notifyList.size(), listSize); + + DropIndexEvent dropIndexEvent = (DropIndexEvent) notifyList.get(listSize - 1); + assert dropIndexEvent.getStatus(); + validateDropIndex(newIndex, dropIndexEvent.getIndex()); + + PreDropIndexEvent preDropIndexEvent = (PreDropIndexEvent) (preNotifyList.get(preNotifyList.size() - 1)); + validateDropIndex(newIndex, preDropIndexEvent.getIndex()); + driver.run("alter table tmptbl add partition (b='2011')"); listSize++; assertEquals(notifyList.size(), listSize); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 47eca29..89ccad5 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -146,6 +146,8 @@ import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.api.UnlockRequest; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; @@ -153,17 +155,21 @@ import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.EventCleanerTask; import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; +import org.apache.hadoop.hive.metastore.events.PreAddIndexEvent; import org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.PreAlterIndexEvent; import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; import org.apache.hadoop.hive.metastore.events.PreAuthorizationCallEvent; import org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; import org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.PreDropIndexEvent; import org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreDropTableEvent; import org.apache.hadoop.hive.metastore.events.PreEventContext; @@ -1381,17 +1387,15 @@ private boolean is_table_exists(RawStore ms, String dbname, String name) return (ms.getTable(dbname, name) != null); } - private void drop_table_core(final RawStore ms, final String dbname, final String name, - final boolean deleteData, final EnvironmentContext envContext) - throws NoSuchObjectException, MetaException, IOException, - InvalidObjectException, InvalidInputException { + private boolean drop_table_core(final RawStore ms, final String dbname, final String name, + final boolean deleteData, final EnvironmentContext envContext, + final String indexName) throws NoSuchObjectException, + MetaException, IOException, InvalidObjectException, InvalidInputException { boolean success = false; boolean isExternal = false; Path tblPath = null; List partPaths = null; Table tbl = null; - isExternal = false; - boolean isIndexTable = false; try { ms.openTransaction(); // drop any partitions @@ -1405,8 +1409,8 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin firePreEvent(new PreDropTableEvent(tbl, deleteData, this)); - isIndexTable = isIndexTable(tbl); - if (isIndexTable) { + boolean isIndexTable = isIndexTable(tbl); + if (indexName == null && isIndexTable) { throw new RuntimeException( "The table " + name + " is an index table. Please do drop index instead."); } @@ -1428,7 +1432,8 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin if (tbl.getSd().getLocation() != null) { tblPath = new Path(tbl.getSd().getLocation()); if (!wh.isWritable(tblPath.getParent())) { - throw new MetaException("Table metadata not deleted since " + + String target = indexName == null ? "Table" : "Index table"; + throw new MetaException(target + " metadata not deleted since " + tblPath.getParent() + " is not writable by " + hiveConf.getUser()); } @@ -1439,17 +1444,17 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin tbl.getPartitionKeys(), deleteData && !isExternal); if (!ms.dropTable(dbname, name)) { - throw new MetaException("Unable to drop table"); + String tableName = dbname + "." + name; + throw new MetaException(indexName == null ? "Unable to drop table " + tableName: + "Unable to drop index table " + tableName + " for index " + indexName); } success = ms.commitTransaction(); } finally { if (!success) { ms.rollbackTransaction(); } else if (deleteData && !isExternal) { - boolean ifPurge = false; - if (envContext != null){ - ifPurge = Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); - } + boolean ifPurge = envContext != null && + Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); // Delete the data in the partitions which have other locations deletePartitionData(partPaths, ifPurge); // Delete the data in the table @@ -1462,6 +1467,7 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin listener.onDropTable(dropTableEvent); } } + return success; } /** @@ -1605,8 +1611,7 @@ public void drop_table_with_environment_context(final String dbname, final Strin boolean success = false; Exception ex = null; try { - drop_table_core(getMS(), dbname, name, deleteData, envContext); - success = true; + success = drop_table_core(getMS(), dbname, name, deleteData, envContext, null); } catch (IOException e) { ex = e; throw new MetaException(e.getMessage()); @@ -3161,7 +3166,12 @@ public void alter_index(final String dbname, final String base_table_name, boolean success = false; Exception ex = null; + Index oldIndex = null; try { + oldIndex = get_index_by_name(dbname, base_table_name, index_name); + + firePreEvent(new PreAlterIndexEvent(oldIndex, newIndex, this)); + getMS().alterIndex(dbname, base_table_name, index_name, newIndex); success = true; } catch (InvalidObjectException e) { @@ -3178,6 +3188,10 @@ public void alter_index(final String dbname, final String base_table_name, } } finally { endFunction("alter_index", success, ex, base_table_name); + for (MetaStoreEventListener listener : listeners) { + AlterIndexEvent alterIndexEvent = new AlterIndexEvent(oldIndex, newIndex, success, this); + listener.onAlterIndex(alterIndexEvent); + } } return; } @@ -3723,6 +3737,8 @@ private Index add_index_core(final RawStore ms, final Index index, final Table i try { ms.openTransaction(); + firePreEvent(new PreAddIndexEvent(index, this)); + Index old_index = null; try { old_index = get_index_by_name(index.getDbName(), index @@ -3770,6 +3786,10 @@ private Index add_index_core(final RawStore ms, final Index index, final Table i } ms.rollbackTransaction(); } + for (MetaStoreEventListener listener : listeners) { + AddIndexEvent addIndexEvent = new AddIndexEvent(index, success, this); + listener.onAddIndex(addIndexEvent); + } } } @@ -3804,16 +3824,17 @@ private boolean drop_index_by_name_core(final RawStore ms, MetaException, TException, IOException, InvalidObjectException, InvalidInputException { boolean success = false; + Index index = null; Path tblPath = null; List partPaths = null; try { ms.openTransaction(); // drop the underlying index table - Index index = get_index_by_name(dbName, tblName, indexName); - if (index == null) { - throw new NoSuchObjectException(indexName + " doesn't exist"); - } + index = get_index_by_name(dbName, tblName, indexName); // throws exception if not exists + + firePreEvent(new PreDropIndexEvent(index, this)); + ms.dropIndex(dbName, tblName, indexName); String idxTblName = index.getIndexTableName(); @@ -3834,26 +3855,29 @@ private boolean drop_index_by_name_core(final RawStore ms, } // Drop the partitions and get a list of partition locations which need to be deleted - partPaths = dropPartitionsAndGetLocations(ms, dbName, idxTblName, tblPath, + partPaths = dropPartitionsAndGetLocations(ms, qualified[0], qualified[1], tblPath, tbl.getPartitionKeys(), deleteData); - if (!ms.dropTable(dbName, idxTblName)) { + if (!ms.dropTable(qualified[0], qualified[1])) { throw new MetaException("Unable to drop underlying data table " - + idxTblName + " for index " + idxTblName); + + qualified[0] + "." + qualified[1] + " for index " + indexName); } } success = ms.commitTransaction(); } finally { if (!success) { ms.rollbackTransaction(); - return false; } else if (deleteData && tblPath != null) { deletePartitionData(partPaths); deleteTableData(tblPath); // ok even if the data is not deleted } + for (MetaStoreEventListener listener : listeners) { + DropIndexEvent dropIndexEvent = new DropIndexEvent(index, success, this); + listener.onDropIndex(dropIndexEvent); + } } - return true; + return success; } @Override @@ -3872,7 +3896,7 @@ public Index get_index_by_name(final String dbName, final String tblName, ex = e; rethrowException(e); } finally { - endFunction("drop_index_by_name", ret != null, ex, tblName); + endFunction("get_index_by_name", ret != null, ex, tblName); } return ret; } @@ -5691,7 +5715,7 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, try { isMetaStoreRemote = true; // Server will create new threads up to max as necessary. After an idle - // period, it will destory threads to keep the number of threads in the + // period, it will destroy threads to keep the number of threads in the // pool to min. int minWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMINTHREADS); int maxWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMAXTHREADS); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java index ec1dca2..792ef42 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java @@ -21,6 +21,8 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; @@ -28,6 +30,7 @@ import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; @@ -117,7 +120,27 @@ public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException { * @throws MetaException */ public void onLoadPartitionDone(LoadPartitionDoneEvent partSetDoneEvent) throws MetaException { + } + + /** + * @param indexEvent index event + * @throws MetaException + */ + public void onAddIndex(AddIndexEvent indexEvent) throws MetaException { + } + + /** + * @param indexEvent index event + * @throws MetaException + */ + public void onDropIndex(DropIndexEvent indexEvent) throws MetaException { + } + /** + * @param indexEvent index event + * @throws MetaException + */ + public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException { } @Override diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 637a39a..004966d 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -64,7 +64,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; @@ -92,7 +91,6 @@ import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; @@ -197,7 +195,7 @@ public Configuration getConf() { } /** - * Called whenever this object is instantiated using ReflectionUils, and also + * Called whenever this object is instantiated using ReflectionUtils, and also * on connection retries. In cases of connection retries, conf will usually * contain modified values. */ @@ -2665,7 +2663,7 @@ public void alterTable(String dbname, String name, Table newTable) throw new MetaException("table " + name + " doesn't exist"); } - // For now only alter name, owner, paramters, cols, bucketcols are allowed + // For now only alter name, owner, parameters, cols, bucketcols are allowed oldt.setDatabase(newt.getDatabase()); oldt.setTableName(newt.getTableName().toLowerCase()); oldt.setParameters(newt.getParameters()); @@ -2708,7 +2706,7 @@ public void alterIndex(String dbname, String baseTblName, String name, Index new throw new MetaException("index " + name + " doesn't exist"); } - // For now only alter paramters are allowed + // For now only alter parameters are allowed oldi.setParameters(newi.getParameters()); // commit the changes @@ -2878,7 +2876,7 @@ private void preDropStorageDescriptor(MStorageDescriptor msd) { MColumnDescriptor mcd = msd.getCD(); // Because there is a 1-N relationship between CDs and SDs, // we must set the SD's CD to null first before dropping the storage descriptor - // to satisfy foriegn key constraints. + // to satisfy foreign key constraints. msd.setCD(null); removeUnusedColumnDescriptor(mcd); } @@ -3019,19 +3017,26 @@ public Index getIndex(String dbName, String origTableName, String indexName) } private Index convertToIndex(MIndex mIndex) throws MetaException { - if(mIndex == null) { + if (mIndex == null) { return null; } + MTable origTable = mIndex.getOrigTable(); + MTable indexTable = mIndex.getIndexTable(); + + String[] qualified = MetaStoreUtils.getQualifiedName( + origTable.getDatabase().getName(), indexTable.getTableName()); + String indexTableName = qualified[0] + "." + qualified[1]; + return new Index( mIndex.getIndexName(), mIndex.getIndexHandlerClass(), - mIndex.getOrigTable().getDatabase().getName(), - mIndex.getOrigTable().getTableName(), + origTable.getDatabase().getName(), + origTable.getTableName(), mIndex.getCreateTime(), mIndex.getLastAccessTime(), - mIndex.getIndexTable().getTableName(), - this.convertToStorageDescriptor(mIndex.getSd()), + indexTableName, + convertToStorageDescriptor(mIndex.getSd()), mIndex.getParameters(), mIndex.getDeferredRebuild()); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java new file mode 100644 index 0000000..43ac0aa --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class AddIndexEvent extends ListenerEvent { + + private final Index index; + + public AddIndexEvent(Index index, boolean status, HiveMetaStore.HMSHandler handler) { + super(status, handler); + this.index = index; + } + + public Index getIndex() { + return index; + } +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java new file mode 100644 index 0000000..4a49700 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class AlterIndexEvent extends ListenerEvent { + + private final Index newIndex; + private final Index oldIndex; + + public AlterIndexEvent(Index oldIndex, Index newIndex, boolean status, + HiveMetaStore.HMSHandler handler) { + super(status, handler); + this.oldIndex = oldIndex; + this.newIndex = newIndex; + } + + public Index getOldIndex() { + return oldIndex; + } + + public Index getNewIndex() { + return newIndex; + } +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java new file mode 100644 index 0000000..06f2302 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class DropIndexEvent extends ListenerEvent { + + private final Index index; + + public DropIndexEvent(Index index, boolean status, HiveMetaStore.HMSHandler handler) { + super(status, handler); + this.index = index; + } + + public Index getIndex() { + return index; + } +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java new file mode 100644 index 0000000..baa04a5 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class PreAddIndexEvent extends PreEventContext { + +private final Index table; + + public PreAddIndexEvent(Index table, HiveMetaStore.HMSHandler handler) { + super(PreEventType.ADD_INDEX, handler); + this.table = table; + } + + public Index getIndex() { + return table; + } +} + diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java new file mode 100644 index 0000000..8ab5af8 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class PreAlterIndexEvent extends PreEventContext { + +private final Index newIndex; + private final Index oldIndex; + + public PreAlterIndexEvent(Index oldIndex, Index newIndex, HiveMetaStore.HMSHandler handler) { + super(PreEventType.ALTER_INDEX, handler); + this.oldIndex = oldIndex; + this.newIndex = newIndex; + } + + public Index getOldIndex() { + return oldIndex; + } + + public Index getNewIndex() { + return newIndex; + } +} + diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java new file mode 100644 index 0000000..437e5c1 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class PreDropIndexEvent extends PreEventContext { + + private final Index index; + + public PreDropIndexEvent(Index index, HiveMetaStore.HMSHandler handler) { + super(PreEventType.DROP_INDEX, handler); + this.index = index; + } + + public Index getIndex() { + return index; + } +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java index dbc3247..ee24a35 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java @@ -39,7 +39,10 @@ LOAD_PARTITION_DONE, AUTHORIZATION_API_CALL, READ_TABLE, - READ_DATABASE + READ_DATABASE, + ADD_INDEX, + ALTER_INDEX, + DROP_INDEX } private final PreEventType eventType; diff --git metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java index 37daf52..a3b16d0 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java @@ -23,13 +23,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; @@ -60,52 +63,70 @@ public DummyListener(Configuration config) { @Override public void onConfigChange(ConfigChangeEvent configChange) { - notifyList.add(configChange); + addEvent(configChange); } @Override public void onAddPartition(AddPartitionEvent partition) throws MetaException { - notifyList.add(partition); + addEvent(partition); } @Override public void onCreateDatabase(CreateDatabaseEvent db) throws MetaException { - notifyList.add(db); + addEvent(db); } @Override public void onCreateTable(CreateTableEvent table) throws MetaException { - notifyList.add(table); + addEvent(table); } @Override public void onDropDatabase(DropDatabaseEvent db) throws MetaException { - notifyList.add(db); + addEvent(db); } @Override public void onDropPartition(DropPartitionEvent partition) throws MetaException { - notifyList.add(partition); + addEvent(partition); } @Override public void onDropTable(DropTableEvent table) throws MetaException { - notifyList.add(table); + addEvent(table); } @Override public void onAlterTable(AlterTableEvent event) throws MetaException { - notifyList.add(event); + addEvent(event); } @Override public void onAlterPartition(AlterPartitionEvent event) throws MetaException { - notifyList.add(event); + addEvent(event); } @Override public void onLoadPartitionDone(LoadPartitionDoneEvent partEvent) throws MetaException { - notifyList.add(partEvent); + addEvent(partEvent); } + @Override + public void onAddIndex(AddIndexEvent indexEvent) throws MetaException { + addEvent(indexEvent); + } + + @Override + public void onDropIndex(DropIndexEvent indexEvent) throws MetaException { + addEvent(indexEvent); + } + + @Override + public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException { + addEvent(indexEvent); + } + + private void addEvent(ListenerEvent event) { + notifyList.add(event); + } } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/DummyPreListener.java metastore/src/test/org/apache/hadoop/hive/metastore/DummyPreListener.java index 9a943b2..7ff6f92 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/DummyPreListener.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyPreListener.java @@ -30,7 +30,7 @@ * * DummyPreListener. * - * An implemenation of MetaStorePreEventListener which stores the Events it's seen in a list. + * An implementation of MetaStorePreEventListener which stores the Events it's seen in a list. */ public class DummyPreListener extends MetaStorePreEventListener { diff --git ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out index 19fbf51..7461ba3 100644 --- ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out +++ ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out @@ -76,5 +76,5 @@ PREHOOK: query: show indexes on src_rc_concatenate_test PREHOOK: type: SHOWINDEXES POSTHOOK: query: show indexes on src_rc_concatenate_test POSTHOOK: type: SHOWINDEXES -src_rc_concatenate_test_index src_rc_concatenate_test key default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact +src_rc_concatenate_test_index src_rc_concatenate_test key default.default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: can not do merge because source table default.src_rc_concatenate_test is indexed. diff --git ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out index ffcbcf9..7f8458d 100644 --- ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out +++ ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out @@ -76,7 +76,7 @@ PREHOOK: query: show indexes on src_rc_concatenate_test PREHOOK: type: SHOWINDEXES POSTHOOK: query: show indexes on src_rc_concatenate_test POSTHOOK: type: SHOWINDEXES -src_rc_concatenate_test_index src_rc_concatenate_test key default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact +src_rc_concatenate_test_index src_rc_concatenate_test key default.default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact PREHOOK: query: alter table src_rc_concatenate_test concatenate PREHOOK: type: ALTER_TABLE_MERGE PREHOOK: Input: default@src_rc_concatenate_test @@ -215,7 +215,7 @@ PREHOOK: query: show indexes on src_rc_concatenate_test_part PREHOOK: type: SHOWINDEXES POSTHOOK: query: show indexes on src_rc_concatenate_test_part POSTHOOK: type: SHOWINDEXES -src_rc_concatenate_test_part_index src_rc_concatenate_test_part key default__src_rc_concatenate_test_part_src_rc_concatenate_test_part_index__ compact +src_rc_concatenate_test_part_index src_rc_concatenate_test_part key default.default__src_rc_concatenate_test_part_src_rc_concatenate_test_part_index__ compact PREHOOK: query: alter table src_rc_concatenate_test_part partition (ds='2011') concatenate PREHOOK: type: ALTER_PARTITION_MERGE PREHOOK: Input: default@src_rc_concatenate_test_part diff --git ql/src/test/results/clientpositive/index_auth.q.out ql/src/test/results/clientpositive/index_auth.q.out index 385b639..39a1096 100644 --- ql/src/test/results/clientpositive/index_auth.q.out +++ ql/src/test/results/clientpositive/index_auth.q.out @@ -24,7 +24,7 @@ PREHOOK: query: SHOW INDEXES ON foobar PREHOOK: type: SHOWINDEXES POSTHOOK: query: SHOW INDEXES ON foobar POSTHOOK: type: SHOWINDEXES -srcpart_auth_index foobar key default__foobar_srcpart_auth_index__ bitmap +srcpart_auth_index foobar key default.default__foobar_srcpart_auth_index__ bitmap PREHOOK: query: grant select on table foobar to user hive_test_user PREHOOK: type: GRANT_PRIVILEGE PREHOOK: Output: default@foobar diff --git ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out index cc6a405..eb2f3bc 100644 --- ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out +++ ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out @@ -118,10 +118,10 @@ PREHOOK: query: SHOW INDEXES ON show_idx_full PREHOOK: type: SHOWINDEXES POSTHOOK: query: SHOW INDEXES ON show_idx_full POSTHOOK: type: SHOWINDEXES -idx_1 show_idx_full key default__show_idx_full_idx_1__ compact -idx_2 show_idx_full value1 default__show_idx_full_idx_2__ compact -idx_comment show_idx_full value2 default__show_idx_full_idx_comment__ compact index comment -idx_compound show_idx_full key, value1 default__show_idx_full_idx_compound__ compact +idx_1 show_idx_full key default.default__show_idx_full_idx_1__ compact +idx_2 show_idx_full value1 default.default__show_idx_full_idx_2__ compact +idx_comment show_idx_full value2 default.default__show_idx_full_idx_comment__ compact index comment +idx_compound show_idx_full key, value1 default.default__show_idx_full_idx_compound__ compact PREHOOK: query: EXPLAIN SHOW INDEXES ON show_idx_empty PREHOOK: type: SHOWINDEXES POSTHOOK: query: EXPLAIN SHOW INDEXES ON show_idx_empty diff --git ql/src/test/results/clientpositive/show_indexes_syntax.q.out ql/src/test/results/clientpositive/show_indexes_syntax.q.out index bc96359..e3cdd09 100644 --- ql/src/test/results/clientpositive/show_indexes_syntax.q.out +++ ql/src/test/results/clientpositive/show_indexes_syntax.q.out @@ -53,7 +53,7 @@ PREHOOK: query: SHOW INDEX ON show_idx_t1 PREHOOK: type: SHOWINDEXES POSTHOOK: query: SHOW INDEX ON show_idx_t1 POSTHOOK: type: SHOWINDEXES -idx_t1 show_idx_t1 key default__show_idx_t1_idx_t1__ compact +idx_t1 show_idx_t1 key default.default__show_idx_t1_idx_t1__ compact PREHOOK: query: EXPLAIN SHOW INDEXES ON show_idx_t1 PREHOOK: type: SHOWINDEXES @@ -79,7 +79,7 @@ PREHOOK: query: SHOW INDEXES ON show_idx_t1 PREHOOK: type: SHOWINDEXES POSTHOOK: query: SHOW INDEXES ON show_idx_t1 POSTHOOK: type: SHOWINDEXES -idx_t1 show_idx_t1 key default__show_idx_t1_idx_t1__ compact +idx_t1 show_idx_t1 key default.default__show_idx_t1_idx_t1__ compact PREHOOK: query: EXPLAIN SHOW FORMATTED INDEXES ON show_idx_t1 PREHOOK: type: SHOWINDEXES @@ -108,7 +108,7 @@ POSTHOOK: type: SHOWINDEXES idx_name tab_name col_names idx_tab_name idx_type comment -idx_t1 show_idx_t1 key default__show_idx_t1_idx_t1__ compact +idx_t1 show_idx_t1 key default.default__show_idx_t1_idx_t1__ compact PREHOOK: query: DROP TABLE show_idx_t1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@show_idx_t1