Index: metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (revision 1138144) +++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (working copy) @@ -24,6 +24,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; @@ -75,6 +77,16 @@ } @Override + public void onAlterTable(AlterTableEvent event) throws MetaException { + notifyList.add(event); + } + + @Override + public void onAlterPartition(AlterPartitionEvent event) throws MetaException { + notifyList.add(event); + } + + @Override public void onLoadPartitionDone(LoadPartitionDoneEvent partEvent) throws MetaException { notifyList.add(partEvent); } Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java (revision 1138144) +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.metastore; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -28,10 +29,13 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; @@ -47,7 +51,6 @@ * {@link org.apache.hadoop.hive.metastore.hadooorg.apache.hadoop.hive.metastore.MetaStoreEventListener} */ public class TestMetaStoreEventListener extends TestCase { - private static final String msPort = "20001"; private HiveConf hiveConf; private HiveMetaStoreClient msc; @@ -93,64 +96,142 @@ } public void testListener() throws Exception { + String dbName = "tmpdb"; + String tblName = "tmptbl"; + String renamed = "tmptbl2"; + int listSize = 0; List notifyList = DummyListener.notifyList; - assertEquals(notifyList.size(), 0); + assertEquals(notifyList.size(), listSize); - driver.run("create database tmpdb"); - Database db = msc.getDatabase("tmpdb"); - assertEquals(1, notifyList.size()); - CreateDatabaseEvent dbEvent = (CreateDatabaseEvent)(notifyList.get(0)); + driver.run("create database " + dbName); + listSize++; + Database db = msc.getDatabase(dbName); + assertEquals(listSize, notifyList.size()); + CreateDatabaseEvent dbEvent = (CreateDatabaseEvent)(notifyList.get(listSize - 1)); assert dbEvent.getStatus(); assertEquals(db.getName(), dbEvent.getDatabase().getName()); assertEquals(db.getLocationUri(), dbEvent.getDatabase().getLocationUri()); - driver.run("use tmpdb"); - driver.run("create table tmptbl (a string) partitioned by (b string)"); - Table tbl = msc.getTable("tmpdb", "tmptbl"); - assertEquals(notifyList.size(), 2); - CreateTableEvent tblEvent = (CreateTableEvent)(notifyList.get(1)); + driver.run("use " + dbName); + driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName)); + listSize++; + Table tbl = msc.getTable(dbName, tblName); + assertEquals(notifyList.size(), listSize); + CreateTableEvent tblEvent = (CreateTableEvent)(notifyList.get(listSize - 1)); assert tblEvent.getStatus(); assertEquals(tbl.getTableName(), tblEvent.getTable().getTableName()); assertEquals(tbl.getDbName(), tblEvent.getTable().getDbName()); assertEquals(tbl.getSd().getLocation(), tblEvent.getTable().getSd().getLocation()); driver.run("alter table tmptbl add partition (b='2011')"); + listSize++; Partition part = msc.getPartition("tmpdb", "tmptbl", "b=2011"); - assertEquals(notifyList.size(), 3); - AddPartitionEvent partEvent = (AddPartitionEvent)(notifyList.get(2)); + assertEquals(notifyList.size(), listSize); + AddPartitionEvent partEvent = (AddPartitionEvent)(notifyList.get(listSize-1)); assert partEvent.getStatus(); assertEquals(part, partEvent.getPartition()); + driver.run(String.format("alter table %s touch partition (%s)", tblName, "b='2011'")); + listSize++; + assertEquals(notifyList.size(), listSize); + AlterPartitionEvent alterPartEvent = (AlterPartitionEvent)notifyList.get(listSize - 1); + assert alterPartEvent.getStatus(); + Partition origP = msc.getPartition(dbName, tblName, "b=2011"); + assertEquals(origP.getValues(), alterPartEvent.getOldPartition().getValues()); + assertEquals(origP.getDbName(), alterPartEvent.getOldPartition().getDbName()); + assertEquals(origP.getTableName(), alterPartEvent.getOldPartition().getTableName()); + //the partition did not change, + // so the new partition should be similar to the original partition + assertEquals(origP.getValues(), alterPartEvent.getNewPartition().getValues()); + assertEquals(origP.getDbName(), alterPartEvent.getNewPartition().getDbName()); + assertEquals(origP.getTableName(), alterPartEvent.getNewPartition().getTableName()); + + driver.run(String.format("alter table %s rename to %s", tblName, renamed)); + listSize++; + assertEquals(notifyList.size(), listSize); + Table renamedTable = msc.getTable(dbName, renamed); + AlterTableEvent alterTableE = (AlterTableEvent) notifyList.get(listSize-1); + assert alterTableE.getStatus(); + Table oldTable = alterTableE.getOldTable(); + Table newTable = alterTableE.getNewTable(); + assertEquals(tbl.getDbName(), oldTable.getDbName()); + assertEquals(tbl.getTableName(), oldTable.getTableName()); + assertEquals(tbl.getSd().getLocation(), oldTable.getSd().getLocation()); + assertEquals(renamedTable.getDbName(), newTable.getDbName()); + assertEquals(renamedTable.getTableName(), newTable.getTableName()); + assertEquals(renamedTable.getSd().getLocation(), newTable.getSd().getLocation()); + + //change the table name back + driver.run(String.format("alter table %s rename to %s", renamed, tblName)); + listSize++; + assertEquals(notifyList.size(), listSize); + + driver.run(String.format("alter table %s ADD COLUMNS (c int)", tblName)); + listSize++; + assertEquals(notifyList.size(), listSize); + Table altTable = msc.getTable(dbName, tblName); + alterTableE = (AlterTableEvent) notifyList.get(listSize-1); + assert alterTableE.getStatus(); + oldTable = alterTableE.getOldTable(); + List origCols = tbl.getSd().getCols(); + List oldCols = oldTable.getSd().getCols(); + List altCols = altTable.getSd().getCols(); + List newCols = altTable.getSd().getCols(); + + Iterator i = origCols.iterator(); + Iterator j = oldCols.iterator(); + while (i.hasNext()) { + assertEquals(i.next().getName(), j.next().getName()); + } + + i = altCols.iterator(); + j = newCols.iterator(); + while (i.hasNext()) { + String name = i.next().getName(); + assertEquals(name, j.next().getName()); + } + + newTable = alterTableE.getNewTable(); + assertEquals(tbl.getDbName(), oldTable.getDbName()); + assertEquals(tbl.getTableName(), oldTable.getTableName()); + assertEquals(tbl.getSd().getLocation(), oldTable.getSd().getLocation()); + assertEquals(altTable.getDbName(), newTable.getDbName()); + assertEquals(altTable.getTableName(), newTable.getTableName()); + assertEquals(altTable.getSd().getLocation(), newTable.getSd().getLocation()); + Map kvs = new HashMap(1); kvs.put("b", "2011"); msc.markPartitionForEvent("tmpdb", "tmptbl", kvs, PartitionEventType.LOAD_DONE); - assertEquals(notifyList.size(), 4); - LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent)notifyList.get(3); + listSize++; + assertEquals(notifyList.size(), listSize); + LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent)notifyList.get(listSize - 1); assert partMarkEvent.getStatus(); assertEquals(partMarkEvent.getPartitionName(), kvs); assertEquals(partMarkEvent.getTable().getTableName(), "tmptbl"); - driver.run("alter table tmptbl drop partition (b='2011')"); - assertEquals(notifyList.size(), 5); - DropPartitionEvent dropPart = (DropPartitionEvent)notifyList.get(4); + driver.run(String.format("alter table %s drop partition (b='2011')", tblName)); + listSize++; + assertEquals(notifyList.size(), listSize); + DropPartitionEvent dropPart = (DropPartitionEvent)notifyList.get(listSize - 1); assert dropPart.getStatus(); assertEquals(part.getValues(), dropPart.getPartition().getValues()); assertEquals(part.getDbName(), dropPart.getPartition().getDbName()); assertEquals(part.getTableName(), dropPart.getPartition().getTableName()); - driver.run("drop table tmptbl"); - assertEquals(notifyList.size(), 6); - DropTableEvent dropTbl = (DropTableEvent)notifyList.get(5); + driver.run("drop table " + tblName); + listSize++; + assertEquals(notifyList.size(), listSize); + DropTableEvent dropTbl = (DropTableEvent)notifyList.get(listSize-1); assert dropTbl.getStatus(); assertEquals(tbl.getTableName(), dropTbl.getTable().getTableName()); assertEquals(tbl.getDbName(), dropTbl.getTable().getDbName()); assertEquals(tbl.getSd().getLocation(), dropTbl.getTable().getSd().getLocation()); - - driver.run("drop database tmpdb"); - assertEquals(notifyList.size(), 7); - DropDatabaseEvent dropDB = (DropDatabaseEvent)notifyList.get(6); + driver.run("drop database " + dbName); + listSize++; + assertEquals(notifyList.size(), listSize); + DropDatabaseEvent dropDB = (DropDatabaseEvent)notifyList.get(listSize-1); assert dropDB.getStatus(); assertEquals(db, dropDB.getDatabase()); } Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java (revision 1138144) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java (working copy) @@ -22,6 +22,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; @@ -43,6 +45,7 @@ public MetaStoreEventListener(Configuration config){ this.conf = config; } + /** * @param create table event. * @throws MetaException @@ -59,6 +62,14 @@ * @param add partition event * @throws MetaException */ + + /** + * @param tableEvent alter table event + * @throws MetaException + */ + public abstract void onAlterTable (AlterTableEvent tableEvent) throws MetaException; + + public abstract void onAddPartition (AddPartitionEvent partitionEvent) throws MetaException; /** @@ -68,6 +79,12 @@ public abstract void onDropPartition (DropPartitionEvent partitionEvent) throws MetaException; /** + * @param alter partition event + * @throws MetaException + */ + public abstract void onAlterPartition (AlterPartitionEvent partitionEvent) throws MetaException; + + /** * @param create database event * @throws MetaException */ @@ -94,4 +111,7 @@ public void setConf(Configuration config) { this.conf = config; } + + + } Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1138144) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -71,6 +71,8 @@ import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; @@ -699,9 +701,9 @@ } for (MetaStoreEventListener listener : listeners) { listener.onDropDatabase(new DropDatabaseEvent(db, success, this)); + } } } - } public void drop_database(final String dbName, final boolean deleteData, final boolean cascade) throws NoSuchObjectException, InvalidOperationException, MetaException { @@ -1708,9 +1710,16 @@ new_part.putToParameters(Constants.DDL_TIME, Long.toString(System .currentTimeMillis() / 1000)); } + Partition oldPart = ms.getPartition(db_name, tbl_name, new_part.getValues()); ms.alterPartition(db_name, tbl_name, new_part); + for (MetaStoreEventListener listener : listeners) { + listener.onAlterPartition(new AlterPartitionEvent(oldPart, new_part, true, this)); + } } catch (InvalidObjectException e) { throw new InvalidOperationException("alter is not possible"); + } catch (NoSuchObjectException e){ + //old partition does not exist + throw new InvalidOperationException("alter is not possible"); } } @@ -1795,20 +1804,25 @@ newTable.putToParameters(Constants.DDL_TIME, Long.toString(System .currentTimeMillis() / 1000)); } - - try { - executeWithRetry(new Command() { + Table oldt = get_table(dbname, name); + boolean success = executeWithRetry(new Command() { @Override public Boolean run(RawStore ms) throws Exception { alterHandler.alterTable(ms, wh, dbname, name, newTable); return Boolean.TRUE; } }); + for (MetaStoreEventListener listener : listeners) { + listener.onAlterTable(new AlterTableEvent(oldt, newTable, success, this)); + } } catch (MetaException e) { throw e; } catch (InvalidOperationException e) { throw e; + } catch (NoSuchObjectException e) { + //thrown when the table to be altered does not exist + throw new InvalidOperationException(e.getMessage()); } catch (Exception e) { assert(e instanceof RuntimeException); throw (RuntimeException)e; Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (revision 1138144) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (working copy) @@ -178,6 +178,7 @@ } finally { if (!success) { msdb.rollbackTransaction(); + throw new MetaException("Committing the alter table transaction was not successful."); } if (success && moveData) { // change the file name in hdfs @@ -204,6 +205,5 @@ } } } - } } Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1138144) +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy) @@ -1728,6 +1728,8 @@ } finally { if (!success) { rollbackTransaction(); + throw new MetaException( + "The transaction for alter partition did not commit successfully."); } } } Index: metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java (revision 0) +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java (revision 0) @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.api.Table; + +public class AlterTableEvent extends ListenerEvent { + + private final Table newTable; + private final Table oldTable; + public AlterTableEvent (Table oldTable, Table newTable, boolean status, HMSHandler handler) { + + super (status, handler); + this.oldTable = oldTable; + this.newTable = newTable; + } + + /** + * @return the old table + */ + public Table getOldTable() { + return oldTable; + } + + /** + * @return the new table + */ + public Table getNewTable() { + return newTable; + } +} \ No newline at end of file Index: metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java (revision 0) +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java (revision 0) @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.api.Partition; + +public class AlterPartitionEvent extends ListenerEvent { + + private final Partition oldPart; + private final Partition newPart; + + public AlterPartitionEvent (Partition oldPart, Partition newPart, + boolean status, HMSHandler handler) { + super (status, handler); + this.oldPart = oldPart; + this.newPart = newPart; + } + + /** + * @return the old partition + */ + public Partition getOldPartition() { + return oldPart; + } + + /** + * + * @return the new partition + */ + public Partition getNewPartition() { + return newPart; + } +} \ No newline at end of file