Index: metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (revision 0) +++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (revision 0) @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.thrift.TBase; + +public class DummyListener implements MetaStoreListener{ + + Configuration conf; + public static final List notifyList = new ArrayList(); + + @Override + public void onAddPartition(Partition partition) throws MetaException { + notifyList.add(partition); + } + + @Override + public void onCreateDatabase(Database db) throws MetaException { + notifyList.add(db); + } + + @Override + public void onCreateTable(Table table) throws MetaException { + notifyList.add(table); + } + + @Override + public void onDropDatabase(Database db) throws MetaException { + notifyList.add(db); + } + + @Override + public void onDropPartition(Partition partition) throws MetaException { + notifyList.add(partition); + } + + @Override + public void onDropTable(Table table) throws MetaException { + notifyList.add(table); + } + + @Override + public Configuration getConf() { + return this.conf; + } + + @Override + public void setConf(Configuration config) { + this.conf = config; + } +} Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreListener.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreListener.java (revision 0) +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreListener.java (revision 0) @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.thrift.TBase; + +public class TestMetaStoreListener extends TestCase { + + private static final String msPort = "20001"; + private HiveConf hiveConf; + private HiveMetaStoreClient msc; + private Driver driver; + + private static class RunMS implements Runnable { + + @Override + public void run() { + HiveMetaStore.main(new String[]{msPort}); + } + + } + + @Override + protected void setUp() throws Exception { + + super.setUp(); + System.setProperty(ConfVars.METASTORE_LISTENER.varname, DummyListener.class.getName()); + Thread t = new Thread(new RunMS()); + t.start(); + Thread.sleep(40000); + hiveConf = new HiveConf(this.getClass()); + hiveConf.set("hive.metastore.local", "false"); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); + hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3); + hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); + hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); + hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + SessionState.start(new CliSessionState(hiveConf)); + msc = new HiveMetaStoreClient(hiveConf,null); + driver = new Driver(hiveConf); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + public void testListener() throws Exception{ + + List notifyList = DummyListener.notifyList; + assertEquals(notifyList.size(), 0); + + driver.run("create database tmpdb"); + Database db = msc.getDatabase("tmpdb"); + assertEquals(1, notifyList.size()); + assertEquals(db.getName(),((Database)notifyList.get(0)).getName()); + assertEquals(db.getLocationUri(),((Database)notifyList.get(0)).getLocationUri()); + + driver.run("use tmpdb"); + driver.run("create table tmptbl (a string) partitioned by (b string)"); + Table tbl = msc.getTable("tmpdb", "tmptbl"); + assertEquals(notifyList.size(), 2); + assertEquals(tbl.getTableName(), ((Table)notifyList.get(1)).getTableName()); + assertEquals(tbl.getDbName(), ((Table)notifyList.get(1)).getDbName()); + assertEquals(tbl.getSd().getLocation(), ((Table)notifyList.get(1)).getSd().getLocation()); + + driver.run("alter table tmptbl add partition (b='2011')"); + Partition part = msc.getPartition("tmpdb", "tmptbl", "b=2011"); + assertEquals(notifyList.size(), 3); + assertEquals(part, notifyList.get(2)); + + driver.run("alter table tmptbl drop partition (b='2011')"); + assertEquals(notifyList.size(), 4); + assertEquals(part.getValues(), ((Partition)notifyList.get(3)).getValues()); + assertEquals(part.getDbName(), ((Partition)notifyList.get(3)).getDbName()); + assertEquals(part.getTableName(), ((Partition)notifyList.get(3)).getTableName()); + + driver.run("drop table tmptbl"); + assertEquals(notifyList.size(), 5); + assertEquals(tbl.getTableName(), ((Table)notifyList.get(4)).getTableName()); + assertEquals(tbl.getDbName(), ((Table)notifyList.get(4)).getDbName()); + assertEquals(tbl.getSd().getLocation(), ((Table)notifyList.get(4)).getSd().getLocation()); + + + driver.run("drop database tmpdb"); + assertEquals(notifyList.size(), 6); + assertEquals(db,notifyList.get(5)); + + } +} Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListener.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListener.java (revision 0) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListener.java (revision 0) @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; + +public interface MetaStoreListener extends Configurable{ + + public void onCreateTable(Table table) throws MetaException; + + public void onDropTable(Table table) throws MetaException; + + public void onAddPartition(Partition partition) throws MetaException; + + public void onDropPartition(Partition partition) throws MetaException; + + public void onCreateDatabase(Database db) throws MetaException; + + public void onDropDatabase(Database db) throws MetaException; + +} Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1079575) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.metrics.Metrics; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException; import org.apache.hadoop.hive.metastore.api.Constants; @@ -203,6 +204,8 @@ private ClassLoader classLoader; private AlterHandler alterHandler; + private MetaStoreListener listener; + { classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { @@ -242,9 +245,25 @@ } } + listener = getMetaStoreListener(hiveConf); return true; } + private MetaStoreListener getMetaStoreListener(HiveConf conf) + throws MetaException{ + + String listenerImpl = conf.getVar(ConfVars.METASTORE_LISTENER); + try { + MetaStoreListener listener = (MetaStoreListener) + ReflectionUtils.newInstance( + Class.forName(listenerImpl, true, JavaUtils.getClassLoader()),conf); + listener.setConf(conf); + return listener; + } catch (ClassNotFoundException e) { + throw new MetaException(e.getMessage()); + } + } + private String addPrefix(String s) { return threadLocalId.get() + ": " + s; } @@ -546,6 +565,7 @@ ms.rollbackTransaction(); } else { wh.mkdirs(new Path(db.getLocationUri())); + this.listener.onCreateDatabase(db); } } } @@ -654,6 +674,9 @@ wh.deleteDir(new Path(db.getLocationUri()), true); // it is not a terrible thing even if the data is not deleted } + if(success) { + this.listener.onDropDatabase(db); + } } } @@ -929,6 +952,9 @@ wh.deleteDir(tblPath, true); } } + else { + this.listener.onCreateTable(tbl); + } } } @@ -1011,7 +1037,6 @@ if (!ms.dropTable(dbname, name)) { throw new MetaException("Unable to drop table"); } - tbl = null; // table collections disappear after dropping success = ms.commitTransaction(); } finally { if (!success) { @@ -1020,6 +1045,9 @@ wh.deleteDir(tblPath, true); // ok even if the data is not deleted } + if(success) { + this.listener.onDropTable(tbl); + } } } @@ -1329,6 +1357,9 @@ wh.deleteDir(partLocation, true); } } + if(success) { + this.listener.onAddPartition(part); + } } return part; } @@ -1410,6 +1441,9 @@ // ok even if the data is not deleted } } + if(success) { + this.listener.onDropPartition(part); + } } return true; } Index: metastore/src/java/org/apache/hadoop/hive/metastore/NoOpListener.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/NoOpListener.java (revision 0) +++ metastore/src/java/org/apache/hadoop/hive/metastore/NoOpListener.java (revision 0) @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; + +public class NoOpListener implements MetaStoreListener { + + private Configuration conf; + + @Override + public void onAddPartition(Partition partition) throws MetaException {} + + @Override + public void onCreateTable(Table table) throws MetaException {} + + @Override + public void onDropPartition(Partition partition) throws MetaException {} + + @Override + public void onDropTable(Table table) throws MetaException {} + + @Override + public void onCreateDatabase(Database db) throws MetaException {} + + @Override + public void onDropDatabase(Database db) throws MetaException {} + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration config) { + this.conf = config; + } +} Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1079575) +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy) @@ -26,9 +26,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Properties; import java.util.Set; +import java.util.Map.Entry; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -86,9 +86,9 @@ import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MTablePrivilege; import org.apache.hadoop.hive.metastore.model.MType; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream; import org.apache.hadoop.hive.metastore.parser.FilterLexer; import org.apache.hadoop.hive.metastore.parser.FilterParser; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream; import org.apache.hadoop.util.StringUtils; /** @@ -994,6 +994,7 @@ throw new NoSuchObjectException("partition values=" + part_vals.toString()); } + part.setValues(part_vals); return part; } Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1079575) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -177,6 +177,7 @@ METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal", ""), METASTORE_USE_THRIFT_SASL("hive.metastore.sasl.enabled", false), METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"), + METASTORE_LISTENER("hive.metastore.listener", "org.apache.hadoop.hive.metastore.NoOpListener"), // Default parameters for creating tables NEWTABLEDEFAULTPARA("hive.table.parameters.default",""),