diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java index db47f9db8b..101e34faf0 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java @@ -41,6 +41,9 @@ private static final org.apache.thrift.protocol.TField TXNID_FIELD_DESC = new org.apache.thrift.protocol.TField("txnid", org.apache.thrift.protocol.TType.I64, (short)1); private static final org.apache.thrift.protocol.TField REPL_POLICY_FIELD_DESC = new org.apache.thrift.protocol.TField("replPolicy", org.apache.thrift.protocol.TType.STRING, (short)2); private static final org.apache.thrift.protocol.TField WRITE_EVENT_INFOS_FIELD_DESC = new org.apache.thrift.protocol.TField("writeEventInfos", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("tableId", org.apache.thrift.protocol.TType.I64, (short)6); + private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)7); + private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)8); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -51,12 +54,18 @@ private long txnid; // required private String replPolicy; // optional private List writeEventInfos; // optional + private long tableId; // optional + private String key; // optional + private String value; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { TXNID((short)1, "txnid"), REPL_POLICY((short)2, "replPolicy"), - WRITE_EVENT_INFOS((short)3, "writeEventInfos"); + WRITE_EVENT_INFOS((short)3, "writeEventInfos"), + TABLE_ID((short)6, "tableId"), + KEY((short)7, "key"), + VALUE((short)8, "value"); private static final Map byName = new HashMap(); @@ -77,6 +86,12 @@ public static _Fields findByThriftId(int fieldId) { return REPL_POLICY; case 3: // WRITE_EVENT_INFOS return WRITE_EVENT_INFOS; + case 6: // TABLE_ID + return TABLE_ID; + case 7: // KEY + return KEY; + case 8: // VALUE + return VALUE; default: return null; } @@ -118,8 +133,9 @@ public String getFieldName() { // isset id assignments private static final int __TXNID_ISSET_ID = 0; + private static final int __TABLEID_ISSET_ID = 1; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.REPL_POLICY,_Fields.WRITE_EVENT_INFOS}; + private static final _Fields optionals[] = {_Fields.REPL_POLICY,_Fields.WRITE_EVENT_INFOS,_Fields.TABLE_ID,_Fields.KEY,_Fields.VALUE}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -130,6 +146,12 @@ public String getFieldName() { tmpMap.put(_Fields.WRITE_EVENT_INFOS, new org.apache.thrift.meta_data.FieldMetaData("writeEventInfos", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT , "WriteEventInfo")))); + tmpMap.put(_Fields.TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("tableId", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CommitTxnRequest.class, metaDataMap); } @@ -161,6 +183,13 @@ public CommitTxnRequest(CommitTxnRequest other) { } this.writeEventInfos = __this__writeEventInfos; } + this.tableId = other.tableId; + if (other.isSetKey()) { + this.key = other.key; + } + if (other.isSetValue()) { + this.value = other.value; + } } public CommitTxnRequest deepCopy() { @@ -173,6 +202,10 @@ public void clear() { this.txnid = 0; this.replPolicy = null; this.writeEventInfos = null; + setTableIdIsSet(false); + this.tableId = 0; + this.key = null; + this.value = null; } public long getTxnid() { @@ -258,6 +291,74 @@ public void setWriteEventInfosIsSet(boolean value) { } } + public long getTableId() { + return this.tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + setTableIdIsSet(true); + } + + public void unsetTableId() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TABLEID_ISSET_ID); + } + + /** Returns true if field tableId is set (has been assigned a value) and false otherwise */ + public boolean isSetTableId() { + return EncodingUtils.testBit(__isset_bitfield, __TABLEID_ISSET_ID); + } + + public void setTableIdIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TABLEID_ISSET_ID, value); + } + + public String getKey() { + return this.key; + } + + public void setKey(String key) { + this.key = key; + } + + public void unsetKey() { + this.key = null; + } + + /** Returns true if field key is set (has been assigned a value) and false otherwise */ + public boolean isSetKey() { + return this.key != null; + } + + public void setKeyIsSet(boolean value) { + if (!value) { + this.key = null; + } + } + + public String getValue() { + return this.value; + } + + public void setValue(String value) { + this.value = value; + } + + public void unsetValue() { + this.value = null; + } + + /** Returns true if field value is set (has been assigned a value) and false otherwise */ + public boolean isSetValue() { + return this.value != null; + } + + public void setValueIsSet(boolean value) { + if (!value) { + this.value = null; + } + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case TXNID: @@ -284,6 +385,30 @@ public void setFieldValue(_Fields field, Object value) { } break; + case TABLE_ID: + if (value == null) { + unsetTableId(); + } else { + setTableId((Long)value); + } + break; + + case KEY: + if (value == null) { + unsetKey(); + } else { + setKey((String)value); + } + break; + + case VALUE: + if (value == null) { + unsetValue(); + } else { + setValue((String)value); + } + break; + } } @@ -298,6 +423,15 @@ public Object getFieldValue(_Fields field) { case WRITE_EVENT_INFOS: return getWriteEventInfos(); + case TABLE_ID: + return getTableId(); + + case KEY: + return getKey(); + + case VALUE: + return getValue(); + } throw new IllegalStateException(); } @@ -315,6 +449,12 @@ public boolean isSet(_Fields field) { return isSetReplPolicy(); case WRITE_EVENT_INFOS: return isSetWriteEventInfos(); + case TABLE_ID: + return isSetTableId(); + case KEY: + return isSetKey(); + case VALUE: + return isSetValue(); } throw new IllegalStateException(); } @@ -359,6 +499,33 @@ public boolean equals(CommitTxnRequest that) { return false; } + boolean this_present_tableId = true && this.isSetTableId(); + boolean that_present_tableId = true && that.isSetTableId(); + if (this_present_tableId || that_present_tableId) { + if (!(this_present_tableId && that_present_tableId)) + return false; + if (this.tableId != that.tableId) + return false; + } + + boolean this_present_key = true && this.isSetKey(); + boolean that_present_key = true && that.isSetKey(); + if (this_present_key || that_present_key) { + if (!(this_present_key && that_present_key)) + return false; + if (!this.key.equals(that.key)) + return false; + } + + boolean this_present_value = true && this.isSetValue(); + boolean that_present_value = true && that.isSetValue(); + if (this_present_value || that_present_value) { + if (!(this_present_value && that_present_value)) + return false; + if (!this.value.equals(that.value)) + return false; + } + return true; } @@ -381,6 +548,21 @@ public int hashCode() { if (present_writeEventInfos) list.add(writeEventInfos); + boolean present_tableId = true && (isSetTableId()); + list.add(present_tableId); + if (present_tableId) + list.add(tableId); + + boolean present_key = true && (isSetKey()); + list.add(present_key); + if (present_key) + list.add(key); + + boolean present_value = true && (isSetValue()); + list.add(present_value); + if (present_value) + list.add(value); + return list.hashCode(); } @@ -422,6 +604,36 @@ public int compareTo(CommitTxnRequest other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetTableId()).compareTo(other.isSetTableId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableId, other.tableId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetKey()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -465,6 +677,32 @@ public String toString() { } first = false; } + if (isSetTableId()) { + if (!first) sb.append(", "); + sb.append("tableId:"); + sb.append(this.tableId); + first = false; + } + if (isSetKey()) { + if (!first) sb.append(", "); + sb.append("key:"); + if (this.key == null) { + sb.append("null"); + } else { + sb.append(this.key); + } + first = false; + } + if (isSetValue()) { + if (!first) sb.append(", "); + sb.append("value:"); + if (this.value == null) { + sb.append("null"); + } else { + sb.append(this.value); + } + first = false; + } sb.append(")"); return sb.toString(); } @@ -549,6 +787,30 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, CommitTxnRequest st org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 6: // TABLE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.tableId = iprot.readI64(); + struct.setTableIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 7: // KEY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.key = iprot.readString(); + struct.setKeyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 8: // VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.value = iprot.readString(); + struct.setValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -586,6 +848,25 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, CommitTxnRequest s oprot.writeFieldEnd(); } } + if (struct.isSetTableId()) { + oprot.writeFieldBegin(TABLE_ID_FIELD_DESC); + oprot.writeI64(struct.tableId); + oprot.writeFieldEnd(); + } + if (struct.key != null) { + if (struct.isSetKey()) { + oprot.writeFieldBegin(KEY_FIELD_DESC); + oprot.writeString(struct.key); + oprot.writeFieldEnd(); + } + } + if (struct.value != null) { + if (struct.isSetValue()) { + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeString(struct.value); + oprot.writeFieldEnd(); + } + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -611,7 +892,16 @@ public void write(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest st if (struct.isSetWriteEventInfos()) { optionals.set(1); } - oprot.writeBitSet(optionals, 2); + if (struct.isSetTableId()) { + optionals.set(2); + } + if (struct.isSetKey()) { + optionals.set(3); + } + if (struct.isSetValue()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); if (struct.isSetReplPolicy()) { oprot.writeString(struct.replPolicy); } @@ -624,6 +914,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest st } } } + if (struct.isSetTableId()) { + oprot.writeI64(struct.tableId); + } + if (struct.isSetKey()) { + oprot.writeString(struct.key); + } + if (struct.isSetValue()) { + oprot.writeString(struct.value); + } } @Override @@ -631,7 +930,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest str TTupleProtocol iprot = (TTupleProtocol) prot; struct.txnid = iprot.readI64(); struct.setTxnidIsSet(true); - BitSet incoming = iprot.readBitSet(2); + BitSet incoming = iprot.readBitSet(5); if (incoming.get(0)) { struct.replPolicy = iprot.readString(); struct.setReplPolicyIsSet(true); @@ -650,6 +949,18 @@ public void read(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest str } struct.setWriteEventInfosIsSet(true); } + if (incoming.get(2)) { + struct.tableId = iprot.readI64(); + struct.setTableIdIsSet(true); + } + if (incoming.get(3)) { + struct.key = iprot.readString(); + struct.setKeyIsSet(true); + } + if (incoming.get(4)) { + struct.value = iprot.readString(); + struct.setValueIsSet(true); + } } } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php index 22deffe1d3..4891e12e14 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php @@ -17248,6 +17248,18 @@ class CommitTxnRequest { * @var \metastore\WriteEventInfo[] */ public $writeEventInfos = null; + /** + * @var int + */ + public $tableId = null; + /** + * @var string + */ + public $key = null; + /** + * @var string + */ + public $value = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -17269,6 +17281,18 @@ class CommitTxnRequest { 'class' => '\metastore\WriteEventInfo', ), ), + 6 => array( + 'var' => 'tableId', + 'type' => TType::I64, + ), + 7 => array( + 'var' => 'key', + 'type' => TType::STRING, + ), + 8 => array( + 'var' => 'value', + 'type' => TType::STRING, + ), ); } if (is_array($vals)) { @@ -17281,6 +17305,15 @@ class CommitTxnRequest { if (isset($vals['writeEventInfos'])) { $this->writeEventInfos = $vals['writeEventInfos']; } + if (isset($vals['tableId'])) { + $this->tableId = $vals['tableId']; + } + if (isset($vals['key'])) { + $this->key = $vals['key']; + } + if (isset($vals['value'])) { + $this->value = $vals['value']; + } } } @@ -17335,6 +17368,27 @@ class CommitTxnRequest { $xfer += $input->skip($ftype); } break; + case 6: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->tableId); + } else { + $xfer += $input->skip($ftype); + } + break; + case 7: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->key); + } else { + $xfer += $input->skip($ftype); + } + break; + case 8: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->value); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -17375,6 +17429,21 @@ class CommitTxnRequest { } $xfer += $output->writeFieldEnd(); } + if ($this->tableId !== null) { + $xfer += $output->writeFieldBegin('tableId', TType::I64, 6); + $xfer += $output->writeI64($this->tableId); + $xfer += $output->writeFieldEnd(); + } + if ($this->key !== null) { + $xfer += $output->writeFieldBegin('key', TType::STRING, 7); + $xfer += $output->writeString($this->key); + $xfer += $output->writeFieldEnd(); + } + if ($this->value !== null) { + $xfer += $output->writeFieldBegin('value', TType::STRING, 8); + $xfer += $output->writeString($this->value); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 38fac465d7..272a2d7868 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -12013,6 +12013,9 @@ class CommitTxnRequest: - txnid - replPolicy - writeEventInfos + - tableId + - key + - value """ thrift_spec = ( @@ -12020,12 +12023,20 @@ class CommitTxnRequest: (1, TType.I64, 'txnid', None, None, ), # 1 (2, TType.STRING, 'replPolicy', None, None, ), # 2 (3, TType.LIST, 'writeEventInfos', (TType.STRUCT,(WriteEventInfo, WriteEventInfo.thrift_spec)), None, ), # 3 + None, # 4 + None, # 5 + (6, TType.I64, 'tableId', None, None, ), # 6 + (7, TType.STRING, 'key', None, None, ), # 7 + (8, TType.STRING, 'value', None, None, ), # 8 ) - def __init__(self, txnid=None, replPolicy=None, writeEventInfos=None,): + def __init__(self, txnid=None, replPolicy=None, writeEventInfos=None, tableId=None, key=None, value=None,): self.txnid = txnid self.replPolicy = replPolicy self.writeEventInfos = writeEventInfos + self.tableId = tableId + self.key = key + self.value = value def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -12057,6 +12068,21 @@ def read(self, iprot): iprot.readListEnd() else: iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.key = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.value = iprot.readString() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -12082,6 +12108,18 @@ def write(self, oprot): iter536.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() + if self.tableId is not None: + oprot.writeFieldBegin('tableId', TType.I64, 6) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + if self.key is not None: + oprot.writeFieldBegin('key', TType.STRING, 7) + oprot.writeString(self.key) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin('value', TType.STRING, 8) + oprot.writeString(self.value) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -12096,6 +12134,9 @@ def __hash__(self): value = (value * 31) ^ hash(self.txnid) value = (value * 31) ^ hash(self.replPolicy) value = (value * 31) ^ hash(self.writeEventInfos) + value = (value * 31) ^ hash(self.tableId) + value = (value * 31) ^ hash(self.key) + value = (value * 31) ^ hash(self.value) return value def __repr__(self): diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb index 0192c6da31..aa9bbb77db 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -2673,11 +2673,17 @@ class CommitTxnRequest TXNID = 1 REPLPOLICY = 2 WRITEEVENTINFOS = 3 + TABLEID = 6 + KEY = 7 + VALUE = 8 FIELDS = { TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid'}, REPLPOLICY => {:type => ::Thrift::Types::STRING, :name => 'replPolicy', :optional => true}, - WRITEEVENTINFOS => {:type => ::Thrift::Types::LIST, :name => 'writeEventInfos', :element => {:type => ::Thrift::Types::STRUCT, :class => ::WriteEventInfo}, :optional => true} + WRITEEVENTINFOS => {:type => ::Thrift::Types::LIST, :name => 'writeEventInfos', :element => {:type => ::Thrift::Types::STRUCT, :class => ::WriteEventInfo}, :optional => true}, + TABLEID => {:type => ::Thrift::Types::I64, :name => 'tableId', :optional => true}, + KEY => {:type => ::Thrift::Types::STRING, :name => 'key', :optional => true}, + VALUE => {:type => ::Thrift::Types::STRING, :name => 'value', :optional => true} } def struct_fields; FIELDS; end diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index df6d56b679..3fd2a124a5 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -50,6 +50,7 @@ import javax.security.auth.login.LoginException; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -2816,6 +2817,22 @@ public void commitTxn(long txnid) client.commit_txn(new CommitTxnRequest(txnid)); } + @Override + public void commitTxnWithKeyValue(long txnid, long tableId, String key, + String value) throws NoSuchTxnException, + TxnAbortedException, TException { + CommitTxnRequest ctr = new CommitTxnRequest(txnid); + Preconditions.checkNotNull(key, "The key to commit together" + + " with the transaction can't be null"); + Preconditions.checkNotNull(value, "The value to commit together" + + " with the transaction can't be null"); + ctr.setTableId(tableId); + ctr.setKey(key); + ctr.setValue(value); + + client.commit_txn(ctr); + } + @Override public void replCommitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbortedException, TException { diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 54e7eda0da..db17e9e412 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -2925,6 +2925,28 @@ Function getFunction(String catName, String dbName, String funcName) void commitTxn(long txnid) throws NoSuchTxnException, TxnAbortedException, TException; + /** + * Like commitTxn but it will atomically store as well a key and a value. This + * can be useful for example to know if a transaction has + * already been committed. TABLE_PARAMS from the metastore must already + * have a row with the TBL_ID corresponding to the table in the parameters + * and PARAM_KEY the same as key in the parameters. The way to update this + * table is with an ALTER command to overwrite/create the table properties. + * @param txnid id of transaction to be committed. + * @param tableId id of the table to associate the key/value with + * @param key key to be committed. + * @param value value to be committed. + * @throws NoSuchTxnException if the requested transaction does not exist. + * This can result fro the transaction having timed out and been deleted by + * the compactor. + * @throws TxnAbortedException if the requested transaction has been + * aborted. This can result from the transaction timing out. + * @throws TException + */ + void commitTxnWithKeyValue(long txnid, long tableId, + String key, String value) throws NoSuchTxnException, + TxnAbortedException, TException; + /** * Commit a transaction. This will also unlock any locks associated with * this transaction. diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 85a5c601e0..7e590780ca 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -901,6 +901,12 @@ struct CommitTxnRequest { 2: optional string replPolicy, // Information related to write operations done in this transaction. 3: optional list writeEventInfos, + + // An optional key/value to store atomically with the transaction + // If the key and value are set, the catalog, database and table must be set as well + 6: optional i64 tableId, + 7: optional string key, + 8: optional string value, } struct WriteEventInfo { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index d76049eda1..dfc1a49233 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -1088,6 +1088,25 @@ public void commitTxn(CommitTxnRequest rqst) LOG.debug("Going to execute update <" + s + ">"); stmt.executeUpdate(s); + // update the key/value associated with the transaction if it has been + // set + if (rqst.isSetKey() && rqst.isSetValue()) { + if (!rqst.isSetTableId()) { + throw new SQLException("If key/value are set, the database and" + + " table must be as well"); + } + s = "UPDATE TABLE_PARAMS SET" + + " PARAM_VALUE = " + quoteString(rqst.getValue()) + + " WHERE TBL_ID = " + rqst.getTableId() + + " AND PARAM_KEY = " + quoteString(rqst.getKey()); + LOG.debug("Going to execute update <" + s + ">"); + int affectedRows = stmt.executeUpdate(s); + if (affectedRows != 1) { + throw new SQLException("Error updating the key/value in the sql" + + " backend. One row should have been affected"); + } + } + if (transactionalListeners != null) { MetaStoreListenerNotifier.notifyEventWithDirectSql(transactionalListeners, EventMessage.EventType.COMMIT_TXN, new CommitTxnEvent(txnid, null), dbConn, sqlGenerator); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index ce590d0f55..aa2011aaa2 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -50,6 +50,7 @@ import javax.security.auth.login.LoginException; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -2277,6 +2278,22 @@ public void commitTxn(long txnid) client.commit_txn(new CommitTxnRequest(txnid)); } + @Override + public void commitTxnWithKeyValue(long txnid, long tableId, String key, + String value) throws NoSuchTxnException, + TxnAbortedException, TException { + CommitTxnRequest ctr = new CommitTxnRequest(txnid); + Preconditions.checkNotNull(key, "The key to commit together" + + " with the transaction can't be null"); + Preconditions.checkNotNull(value, "The value to commit together" + + " with the transaction can't be null"); + ctr.setTableId(tableId); + ctr.setKey(key); + ctr.setValue(value); + + client.commit_txn(ctr); + } + @Override public void replCommitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbortedException, TException { diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java index db4dd9ec42..d6ad3b0354 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse; import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.LockState; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.junit.After; @@ -33,6 +34,11 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.List; /** @@ -52,6 +58,7 @@ private final Configuration conf = MetastoreConf.newMetastoreConf(); private IMetaStoreClient client; + private Connection conn; @Test public void testTxns() throws Exception { @@ -83,6 +90,54 @@ public void testOpenTxnNotExcluded() throws Exception { Assert.assertFalse(validTxns.isTxnValid(4)); } + private void cleanUpParamsTable() throws SQLException { + Statement stm = conn.createStatement(); + stm.execute("DELETE FROM TABLE_PARAMS WHERE TBL_ID = 200 AND PARAM_KEY = 'mykey'"); + stm.execute("DELETE FROM TBLS WHERE TBL_ID = 200"); + stm.execute("DELETE FROM DBS WHERE DB_ID = 20"); + } + + @Test + public void testTxNWithKeyValue() throws Exception { + Statement stm = conn.createStatement(); + stm.executeUpdate("INSERT INTO DBS(DB_ID, NAME, CTLG_NAME," + + " DB_LOCATION_URI) VALUES(20, 'mydb', 'hive', '/')"); + stm.executeUpdate("INSERT INTO TBLS(TBL_ID, DB_ID, TBL_NAME," + + " CREATE_TIME, LAST_ACCESS_TIME, RETENTION)" + + " VALUES(200, 20, 'mytable', 10, 10, 10)"); + stm.executeUpdate("INSERT INTO TABLE_PARAMS(TBL_ID, PARAM_KEY)" + + " VALUES(200, 'mykey')"); + + List tids = client.openTxns("me", 1).getTxn_ids(); + Assert.assertEquals(1L, (long) tids.get(0)); + client.commitTxnWithKeyValue(1, 200, "mykey", "myvalue"); + ValidTxnList validTxns = client.getValidTxns(1); + Assert.assertTrue(validTxns.isTxnValid(1)); + + ResultSet rs = stm.executeQuery("SELECT TBL_ID, PARAM_KEY, PARAM_VALUE" + + " FROM TABLE_PARAMS WHERE TBL_ID = 200"); + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getLong(1), 200); + Assert.assertEquals(rs.getString(2), "mykey"); + Assert.assertEquals(rs.getString(3), "myvalue"); + Assert.assertFalse(rs.next()); + cleanUpParamsTable(); + } + + @Test + public void testTxNWithKeyValueNoTableId() throws Exception { + List tids = client.openTxns("me", 1).getTxn_ids(); + Assert.assertEquals(1L, (long) tids.get(0)); + try { + client.commitTxnWithKeyValue(1, 10, "mykey", + "myvalue"); + Assert.fail("Should have raised exception"); + } catch (MetaException e) {} + ValidTxnList validTxns = client.getValidTxns(1); + Assert.assertTrue(validTxns.isTxnValid(1)); + cleanUpParamsTable(); + } + @Test public void testTxnRange() throws Exception { ValidTxnList validTxns = client.getValidTxns(); @@ -258,10 +313,14 @@ public void setUp() throws Exception { TxnDbUtil.setConfValues(conf); TxnDbUtil.prepDb(conf); client = new HiveMetaStoreClient(conf); + String connectionStr = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY); + + conn = DriverManager.getConnection(connectionStr); } @After public void tearDown() throws Exception { + conn.close(); TxnDbUtil.cleanDb(conf); } }