Index: metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (revision 1133167) +++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (working copy) @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; +import org.apache.hadoop.hive.metastore.events.PartitionSetDoneEvent; /** A dummy implementation for * {@link org.apache.hadoop.hive.metastore.hadooorg.apache.hadoop.hive.metastore.MetaStoreEventListener} @@ -72,4 +73,9 @@ public void onDropTable(DropTableEvent table) throws MetaException { notifyList.add(table); } + + @Override + public void onMarkingPartitionSetDone(PartitionSetDoneEvent partEvent) throws MetaException { + notifyList.add(partEvent); + } } Index: metastore/src/model/package.jdo =================================================================== --- metastore/src/model/package.jdo (revision 1133167) +++ metastore/src/model/package.jdo (working copy) @@ -665,5 +665,20 @@ + + + + + + + + + + + + + + + Index: metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionSet.java =================================================================== --- metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionSet.java (revision 0) +++ metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionSet.java (revision 0) @@ -0,0 +1,66 @@ +package org.apache.hadoop.hive.metastore.model; + +public class MPartitionSet { + + private String dbName; + + private String tblName; + + private String partVals; + + private Long createTime; + + /** + * @return the createTime + */ + public Long getCreateTime() { + return createTime; + } + + public MPartitionSet(String dbName, String tblName, String partVals) { + super(); + this.dbName = dbName; + this.tblName = tblName; + this.partVals = partVals; + this.createTime = System.currentTimeMillis(); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "MPartitionSet [dbName=" + dbName + ", tblName=" + tblName + + ", partSpec=" + partVals + ", createTime=" + createTime + "]"; + } + + public MPartitionSet() {} + + /** + * @param dbName the dbName to set + */ + public void setDbName(String dbName) { + this.dbName = dbName; + } + + /** + * @param tblName the tblName to set + */ + public void setTblName(String tblName) { + this.tblName = tblName; + } + + /** + * @param partSpec the partSpec to set + */ + public void setPartSpec(String partSpec) { + this.partVals = partSpec; + } + + /** + * @param createTime the createTime to set + */ + public void setCreateTime(Long createTime) { + this.createTime = createTime; + } +} Index: metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (revision 1133167) +++ metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (working copy) @@ -149,6 +149,10 @@ String dbName, String tblName, List partNames) throws MetaException, NoSuchObjectException; + public abstract void markPartitionSetDone(String dbName, String tblName, List partVals); + + public abstract boolean isPartitionSetMarkedDone(String dbName, String tblName, List partVals); + public abstract boolean addRole(String rowName, String ownerName) throws InvalidObjectException, MetaException, NoSuchObjectException; Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java (revision 1133167) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java (working copy) @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; +import org.apache.hadoop.hive.metastore.events.PartitionSetDoneEvent; /** * This abstract class needs to be extended to provide implementation of actions that needs @@ -78,6 +79,8 @@ */ public abstract void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException; + public abstract void onMarkingPartitionSetDone(PartitionSetDoneEvent partSetDoneEvent) throws MetaException; + @Override public Configuration getConf() { return this.conf; Index: metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (revision 1133167) +++ metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (working copy) @@ -366,6 +366,33 @@ List groupNames) throws MetaException, TException, NoSuchObjectException; /** + * @param db_name + * @param tbl_name + * @param part_vals + * @throws MetaException + * @throws UnknownTableException + * @throws UnknownDBException + * @throws InvalidObjectException + * @throws TException + */ + public void markPartitionSetDone(String db_name, String tbl_name, List part_vals) + throws MetaException, NoSuchObjectException, TException; + + /** + * @param db_name + * @param tbl_name + * @param part_vals + * @return + * @throws MetaException + * @throws UnknownTableException + * @throws UnknownDBException + * @throws InvalidObjectException + * @throws TException + */ + public boolean isPartitionSetDone(String db_name, String tbl_name, List part_vals) + throws MetaException, NoSuchObjectException, TException; + + /** * @param tbl * @throws AlreadyExistsException * @throws InvalidObjectException @@ -374,6 +401,7 @@ * @throws TException * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#create_table(org.apache.hadoop.hive.metastore.api.Table) */ + public void createTable(Table tbl) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException; Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (revision 1133167) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (working copy) @@ -1064,7 +1064,7 @@ String owner = conf.getUser(); return getDelegationToken(owner, renewerKerberosPrincipalName); } - + @Override public String getDelegationToken(String owner, String renewerKerberosPrincipalName) throws MetaException, TException { @@ -1131,4 +1131,16 @@ } } + @Override + public void markPartitionSetDone(String db_name, String tbl_name, List part_vals) + throws MetaException, TException, NoSuchObjectException { + client.markPartitionSetDone(db_name, tbl_name, part_vals); + } + + @Override + public boolean isPartitionSetDone(String db_name, String tbl_name, List part_vals) + throws MetaException, NoSuchObjectException, TException { + return client.isPartitionSetDone(db_name, tbl_name, part_vals); + } + } Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1133167) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -73,6 +73,7 @@ import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; +import org.apache.hadoop.hive.metastore.events.PartitionSetDoneEvent; import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege; @@ -3120,6 +3121,45 @@ } } + @Override + public void markPartitionSetDone(final String db_name, final String tbl_name, final List part_vals) + throws MetaException,TException, NoSuchObjectException { + Table tbl = get_table(db_name, tbl_name); // Make sure dbName and tblName are valid. + assert tbl.getSd().getCols().size() + tbl.getPartitionKeys().size() == part_vals.size(); + boolean status = false; + try{ + status = executeWithRetry(new Command(){ + @Override + public Boolean run(RawStore ms) throws Exception { + ms.markPartitionSetDone(db_name, tbl_name, part_vals); + return true; + } + }); + } catch(Exception e){ + + } + for(MetaStoreEventListener listener : listeners){ + listener.onMarkingPartitionSetDone(new PartitionSetDoneEvent(status, this, db_name, tbl_name, part_vals)); + } + } + + @Override + public boolean isPartitionSetDone(final String db_name, final String tbl_name, final List part_vals) + throws MetaException, NoSuchObjectException, TException { + Table tbl = get_table(db_name, tbl_name); // Make sure dbName and tblName are valid. + assert tbl.getSd().getCols().size() + tbl.getPartitionKeys().size() == part_vals.size(); + try { + return executeWithRetry(new Command(){ + @Override + public Boolean run(RawStore ms) throws Exception { + return ms.isPartitionSetMarkedDone(db_name, tbl_name, part_vals); + } + + }); + } catch (Exception e) { + throw new MetaException(e.toString()); + } + } } /** Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1133167) +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy) @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.metastore; +import static org.apache.commons.lang.StringUtils.join; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -26,9 +28,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.util.Map.Entry; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -71,6 +73,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.model.MPartitionSet; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MFieldSchema; @@ -88,9 +91,9 @@ import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MTablePrivilege; import org.apache.hadoop.hive.metastore.model.MType; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream; import org.apache.hadoop.hive.metastore.parser.FilterLexer; import org.apache.hadoop.hive.metastore.parser.FilterParser; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream; import org.apache.hadoop.util.StringUtils; /** @@ -3360,4 +3363,27 @@ return mSecurityColumnList; } + @Override + public boolean isPartitionSetMarkedDone(String dbName, String tblName, List partVals) { + + openTransaction(); + Query query = pm.newQuery(MPartitionSet.class, + "dbName == t1 && tblName == t2 && partSpec == t3"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); + query.setUnique(true); + MPartitionSet partSet = (MPartitionSet) query.execute(dbName, tblName, join(partVals,',')); + commitTransaction(); + return partSet != null ? true : false; + + } + + @Override + public void markPartitionSetDone(String dbName, String tblName, List partVals) { + openTransaction(); + pm.makePersistent(new MPartitionSet( + dbName,tblName,join(partVals,','))); + commitTransaction(); + + } + } Index: metastore/src/java/org/apache/hadoop/hive/metastore/events/PartitionSetDoneEvent.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/events/PartitionSetDoneEvent.java (revision 0) +++ metastore/src/java/org/apache/hadoop/hive/metastore/events/PartitionSetDoneEvent.java (revision 0) @@ -0,0 +1,44 @@ +package org.apache.hadoop.hive.metastore.events; + +import java.util.List; + +import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; + +public class PartitionSetDoneEvent extends ListenerEvent { + + private final String dbName; + + private final String tblName; + + private final List partSpec; + + public PartitionSetDoneEvent(boolean status, HMSHandler handler, String dbName, + String tblName, List partSpec) { + super(status, handler); + this.dbName = dbName; + this.tblName = tblName; + this.partSpec = partSpec; + } + + /** + * @return the dbName + */ + public String getDbName() { + return dbName; + } + + /** + * @return the tblName + */ + public String getTblName() { + return tblName; + } + + /** + * @return the partSpec + */ + public List getPartSpec() { + return partSpec; + } + +} Index: metastore/if/hive_metastore.thrift =================================================================== --- metastore/if/hive_metastore.thrift (revision 1133167) +++ metastore/if/hive_metastore.thrift (working copy) @@ -328,6 +328,11 @@ map partition_name_to_spec(1: string part_name) throws(1: MetaException o1) + void markPartitionSetDone(1:string db_name, 2:string tbl_name, 3:list part_vals) + throws (1: MetaException o1, 2: NoSuchObjectException o4) + bool isPartitionSetDone(1:string db_name, 2:string tbl_name, 3:list part_vals) + throws (1: MetaException o1, 2:NoSuchObjectException o2) + //index Index add_index(1:Index new_index, 2: Table index_table) throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)