Index: metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java
===================================================================
--- metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (revision 1133167)
+++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (working copy)
@@ -30,6 +30,7 @@
import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
import org.apache.hadoop.hive.metastore.events.ListenerEvent;
+import org.apache.hadoop.hive.metastore.events.PartitionSetDoneEvent;
/** A dummy implementation for
* {@link org.apache.hadoop.hive.metastore.hadooorg.apache.hadoop.hive.metastore.MetaStoreEventListener}
@@ -72,4 +73,9 @@
public void onDropTable(DropTableEvent table) throws MetaException {
notifyList.add(table);
}
+
+ @Override
+ public void onMarkingPartitionSetDone(PartitionSetDoneEvent partEvent) throws MetaException {
+ notifyList.add(partEvent);
+ }
}
Index: metastore/src/model/package.jdo
===================================================================
--- metastore/src/model/package.jdo (revision 1133167)
+++ metastore/src/model/package.jdo (working copy)
@@ -665,5 +665,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionSet.java
===================================================================
--- metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionSet.java (revision 0)
+++ metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionSet.java (revision 0)
@@ -0,0 +1,66 @@
+package org.apache.hadoop.hive.metastore.model;
+
+public class MPartitionSet {
+
+ private String dbName;
+
+ private String tblName;
+
+ private String partVals;
+
+ private Long createTime;
+
+ /**
+ * @return the createTime
+ */
+ public Long getCreateTime() {
+ return createTime;
+ }
+
+ public MPartitionSet(String dbName, String tblName, String partVals) {
+ super();
+ this.dbName = dbName;
+ this.tblName = tblName;
+ this.partVals = partVals;
+ this.createTime = System.currentTimeMillis();
+ }
+
+ /* (non-Javadoc)
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ return "MPartitionSet [dbName=" + dbName + ", tblName=" + tblName
+ + ", partSpec=" + partVals + ", createTime=" + createTime + "]";
+ }
+
+ public MPartitionSet() {}
+
+ /**
+ * @param dbName the dbName to set
+ */
+ public void setDbName(String dbName) {
+ this.dbName = dbName;
+ }
+
+ /**
+ * @param tblName the tblName to set
+ */
+ public void setTblName(String tblName) {
+ this.tblName = tblName;
+ }
+
+ /**
+ * @param partSpec the partSpec to set
+ */
+ public void setPartSpec(String partSpec) {
+ this.partVals = partSpec;
+ }
+
+ /**
+ * @param createTime the createTime to set
+ */
+ public void setCreateTime(Long createTime) {
+ this.createTime = createTime;
+ }
+}
Index: metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (revision 1133167)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (working copy)
@@ -149,6 +149,10 @@
String dbName, String tblName, List partNames)
throws MetaException, NoSuchObjectException;
+ public abstract void markPartitionSetDone(String dbName, String tblName, List partVals);
+
+ public abstract boolean isPartitionSetMarkedDone(String dbName, String tblName, List partVals);
+
public abstract boolean addRole(String rowName, String ownerName)
throws InvalidObjectException, MetaException, NoSuchObjectException;
Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java (revision 1133167)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java (working copy)
@@ -27,6 +27,7 @@
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.PartitionSetDoneEvent;
/**
* This abstract class needs to be extended to provide implementation of actions that needs
@@ -78,6 +79,8 @@
*/
public abstract void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException;
+ public abstract void onMarkingPartitionSetDone(PartitionSetDoneEvent partSetDoneEvent) throws MetaException;
+
@Override
public Configuration getConf() {
return this.conf;
Index: metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (revision 1133167)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (working copy)
@@ -366,6 +366,33 @@
List groupNames) throws MetaException, TException, NoSuchObjectException;
/**
+ * @param db_name
+ * @param tbl_name
+ * @param part_vals
+ * @throws MetaException
+ * @throws UnknownTableException
+ * @throws UnknownDBException
+ * @throws InvalidObjectException
+ * @throws TException
+ */
+ public void markPartitionSetDone(String db_name, String tbl_name, List part_vals)
+ throws MetaException, NoSuchObjectException, TException;
+
+ /**
+ * @param db_name
+ * @param tbl_name
+ * @param part_vals
+ * @return
+ * @throws MetaException
+ * @throws UnknownTableException
+ * @throws UnknownDBException
+ * @throws InvalidObjectException
+ * @throws TException
+ */
+ public boolean isPartitionSetDone(String db_name, String tbl_name, List part_vals)
+ throws MetaException, NoSuchObjectException, TException;
+
+ /**
* @param tbl
* @throws AlreadyExistsException
* @throws InvalidObjectException
@@ -374,6 +401,7 @@
* @throws TException
* @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#create_table(org.apache.hadoop.hive.metastore.api.Table)
*/
+
public void createTable(Table tbl) throws AlreadyExistsException,
InvalidObjectException, MetaException, NoSuchObjectException, TException;
Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (revision 1133167)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (working copy)
@@ -1064,7 +1064,7 @@
String owner = conf.getUser();
return getDelegationToken(owner, renewerKerberosPrincipalName);
}
-
+
@Override
public String getDelegationToken(String owner, String renewerKerberosPrincipalName) throws
MetaException, TException {
@@ -1131,4 +1131,16 @@
}
}
+ @Override
+ public void markPartitionSetDone(String db_name, String tbl_name, List part_vals)
+ throws MetaException, TException, NoSuchObjectException {
+ client.markPartitionSetDone(db_name, tbl_name, part_vals);
+ }
+
+ @Override
+ public boolean isPartitionSetDone(String db_name, String tbl_name, List part_vals)
+ throws MetaException, NoSuchObjectException, TException {
+ return client.isPartitionSetDone(db_name, tbl_name, part_vals);
+ }
+
}
Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1133167)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy)
@@ -73,6 +73,7 @@
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.PartitionSetDoneEvent;
import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook;
import org.apache.hadoop.hive.metastore.model.MDBPrivilege;
import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege;
@@ -3120,6 +3121,45 @@
}
}
+ @Override
+ public void markPartitionSetDone(final String db_name, final String tbl_name, final List part_vals)
+ throws MetaException,TException, NoSuchObjectException {
+ Table tbl = get_table(db_name, tbl_name); // Make sure dbName and tblName are valid.
+ assert tbl.getSd().getCols().size() + tbl.getPartitionKeys().size() == part_vals.size();
+ boolean status = false;
+ try{
+ status = executeWithRetry(new Command(){
+ @Override
+ public Boolean run(RawStore ms) throws Exception {
+ ms.markPartitionSetDone(db_name, tbl_name, part_vals);
+ return true;
+ }
+ });
+ } catch(Exception e){
+
+ }
+ for(MetaStoreEventListener listener : listeners){
+ listener.onMarkingPartitionSetDone(new PartitionSetDoneEvent(status, this, db_name, tbl_name, part_vals));
+ }
+ }
+
+ @Override
+ public boolean isPartitionSetDone(final String db_name, final String tbl_name, final List part_vals)
+ throws MetaException, NoSuchObjectException, TException {
+ Table tbl = get_table(db_name, tbl_name); // Make sure dbName and tblName are valid.
+ assert tbl.getSd().getCols().size() + tbl.getPartitionKeys().size() == part_vals.size();
+ try {
+ return executeWithRetry(new Command(){
+ @Override
+ public Boolean run(RawStore ms) throws Exception {
+ return ms.isPartitionSetMarkedDone(db_name, tbl_name, part_vals);
+ }
+
+ });
+ } catch (Exception e) {
+ throw new MetaException(e.toString());
+ }
+ }
}
/**
Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1133167)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy)
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.metastore;
+import static org.apache.commons.lang.StringUtils.join;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -26,9 +28,9 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import java.util.Map.Entry;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -71,6 +73,7 @@
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.model.MPartitionSet;
import org.apache.hadoop.hive.metastore.model.MDBPrivilege;
import org.apache.hadoop.hive.metastore.model.MDatabase;
import org.apache.hadoop.hive.metastore.model.MFieldSchema;
@@ -88,9 +91,9 @@
import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege;
import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
import org.apache.hadoop.hive.metastore.model.MType;
+import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream;
import org.apache.hadoop.hive.metastore.parser.FilterLexer;
import org.apache.hadoop.hive.metastore.parser.FilterParser;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream;
import org.apache.hadoop.util.StringUtils;
/**
@@ -3360,4 +3363,27 @@
return mSecurityColumnList;
}
+ @Override
+ public boolean isPartitionSetMarkedDone(String dbName, String tblName, List partVals) {
+
+ openTransaction();
+ Query query = pm.newQuery(MPartitionSet.class,
+ "dbName == t1 && tblName == t2 && partSpec == t3");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
+ query.setUnique(true);
+ MPartitionSet partSet = (MPartitionSet) query.execute(dbName, tblName, join(partVals,','));
+ commitTransaction();
+ return partSet != null ? true : false;
+
+ }
+
+ @Override
+ public void markPartitionSetDone(String dbName, String tblName, List partVals) {
+ openTransaction();
+ pm.makePersistent(new MPartitionSet(
+ dbName,tblName,join(partVals,',')));
+ commitTransaction();
+
+ }
+
}
Index: metastore/src/java/org/apache/hadoop/hive/metastore/events/PartitionSetDoneEvent.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/events/PartitionSetDoneEvent.java (revision 0)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/events/PartitionSetDoneEvent.java (revision 0)
@@ -0,0 +1,44 @@
+package org.apache.hadoop.hive.metastore.events;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+
+public class PartitionSetDoneEvent extends ListenerEvent {
+
+ private final String dbName;
+
+ private final String tblName;
+
+ private final List partSpec;
+
+ public PartitionSetDoneEvent(boolean status, HMSHandler handler, String dbName,
+ String tblName, List partSpec) {
+ super(status, handler);
+ this.dbName = dbName;
+ this.tblName = tblName;
+ this.partSpec = partSpec;
+ }
+
+ /**
+ * @return the dbName
+ */
+ public String getDbName() {
+ return dbName;
+ }
+
+ /**
+ * @return the tblName
+ */
+ public String getTblName() {
+ return tblName;
+ }
+
+ /**
+ * @return the partSpec
+ */
+ public List getPartSpec() {
+ return partSpec;
+ }
+
+}
Index: metastore/if/hive_metastore.thrift
===================================================================
--- metastore/if/hive_metastore.thrift (revision 1133167)
+++ metastore/if/hive_metastore.thrift (working copy)
@@ -328,6 +328,11 @@
map partition_name_to_spec(1: string part_name)
throws(1: MetaException o1)
+ void markPartitionSetDone(1:string db_name, 2:string tbl_name, 3:list part_vals)
+ throws (1: MetaException o1, 2: NoSuchObjectException o4)
+ bool isPartitionSetDone(1:string db_name, 2:string tbl_name, 3:list part_vals)
+ throws (1: MetaException o1, 2:NoSuchObjectException o2)
+
//index
Index add_index(1:Index new_index, 2: Table index_table)
throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)