diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java index db47f9db8b..d89924c155 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java @@ -41,6 +41,11 @@ private static final org.apache.thrift.protocol.TField TXNID_FIELD_DESC = new org.apache.thrift.protocol.TField("txnid", org.apache.thrift.protocol.TType.I64, (short)1); private static final org.apache.thrift.protocol.TField REPL_POLICY_FIELD_DESC = new org.apache.thrift.protocol.TField("replPolicy", org.apache.thrift.protocol.TType.STRING, (short)2); private static final org.apache.thrift.protocol.TField WRITE_EVENT_INFOS_FIELD_DESC = new org.apache.thrift.protocol.TField("writeEventInfos", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField CATALOG_FIELD_DESC = new org.apache.thrift.protocol.TField("catalog", org.apache.thrift.protocol.TType.STRING, (short)4); + private static final org.apache.thrift.protocol.TField DATABASE_FIELD_DESC = new org.apache.thrift.protocol.TField("database", org.apache.thrift.protocol.TType.STRING, (short)5); + private static final org.apache.thrift.protocol.TField TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("table", org.apache.thrift.protocol.TType.STRING, (short)6); + private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)7); + private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)8); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -51,12 +56,22 @@ private long txnid; // required private String replPolicy; // optional private List writeEventInfos; // optional + private String catalog; // optional + private String database; // optional + private String table; // optional + private String key; // optional + private String value; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { TXNID((short)1, "txnid"), REPL_POLICY((short)2, "replPolicy"), - WRITE_EVENT_INFOS((short)3, "writeEventInfos"); + WRITE_EVENT_INFOS((short)3, "writeEventInfos"), + CATALOG((short)4, "catalog"), + DATABASE((short)5, "database"), + TABLE((short)6, "table"), + KEY((short)7, "key"), + VALUE((short)8, "value"); private static final Map byName = new HashMap(); @@ -77,6 +92,16 @@ public static _Fields findByThriftId(int fieldId) { return REPL_POLICY; case 3: // WRITE_EVENT_INFOS return WRITE_EVENT_INFOS; + case 4: // CATALOG + return CATALOG; + case 5: // DATABASE + return DATABASE; + case 6: // TABLE + return TABLE; + case 7: // KEY + return KEY; + case 8: // VALUE + return VALUE; default: return null; } @@ -119,7 +144,7 @@ public String getFieldName() { // isset id assignments private static final int __TXNID_ISSET_ID = 0; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.REPL_POLICY,_Fields.WRITE_EVENT_INFOS}; + private static final _Fields optionals[] = {_Fields.REPL_POLICY,_Fields.WRITE_EVENT_INFOS,_Fields.CATALOG,_Fields.DATABASE,_Fields.TABLE,_Fields.KEY,_Fields.VALUE}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -130,6 +155,16 @@ public String getFieldName() { tmpMap.put(_Fields.WRITE_EVENT_INFOS, new org.apache.thrift.meta_data.FieldMetaData("writeEventInfos", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT , "WriteEventInfo")))); + tmpMap.put(_Fields.CATALOG, new org.apache.thrift.meta_data.FieldMetaData("catalog", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.DATABASE, new org.apache.thrift.meta_data.FieldMetaData("database", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.TABLE, new org.apache.thrift.meta_data.FieldMetaData("table", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CommitTxnRequest.class, metaDataMap); } @@ -161,6 +196,21 @@ public CommitTxnRequest(CommitTxnRequest other) { } this.writeEventInfos = __this__writeEventInfos; } + if (other.isSetCatalog()) { + this.catalog = other.catalog; + } + if (other.isSetDatabase()) { + this.database = other.database; + } + if (other.isSetTable()) { + this.table = other.table; + } + if (other.isSetKey()) { + this.key = other.key; + } + if (other.isSetValue()) { + this.value = other.value; + } } public CommitTxnRequest deepCopy() { @@ -173,6 +223,11 @@ public void clear() { this.txnid = 0; this.replPolicy = null; this.writeEventInfos = null; + this.catalog = null; + this.database = null; + this.table = null; + this.key = null; + this.value = null; } public long getTxnid() { @@ -258,6 +313,121 @@ public void setWriteEventInfosIsSet(boolean value) { } } + public String getCatalog() { + return this.catalog; + } + + public void setCatalog(String catalog) { + this.catalog = catalog; + } + + public void unsetCatalog() { + this.catalog = null; + } + + /** Returns true if field catalog is set (has been assigned a value) and false otherwise */ + public boolean isSetCatalog() { + return this.catalog != null; + } + + public void setCatalogIsSet(boolean value) { + if (!value) { + this.catalog = null; + } + } + + public String getDatabase() { + return this.database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public void unsetDatabase() { + this.database = null; + } + + /** Returns true if field database is set (has been assigned a value) and false otherwise */ + public boolean isSetDatabase() { + return this.database != null; + } + + public void setDatabaseIsSet(boolean value) { + if (!value) { + this.database = null; + } + } + + public String getTable() { + return this.table; + } + + public void setTable(String table) { + this.table = table; + } + + public void unsetTable() { + this.table = null; + } + + /** Returns true if field table is set (has been assigned a value) and false otherwise */ + public boolean isSetTable() { + return this.table != null; + } + + public void setTableIsSet(boolean value) { + if (!value) { + this.table = null; + } + } + + public String getKey() { + return this.key; + } + + public void setKey(String key) { + this.key = key; + } + + public void unsetKey() { + this.key = null; + } + + /** Returns true if field key is set (has been assigned a value) and false otherwise */ + public boolean isSetKey() { + return this.key != null; + } + + public void setKeyIsSet(boolean value) { + if (!value) { + this.key = null; + } + } + + public String getValue() { + return this.value; + } + + public void setValue(String value) { + this.value = value; + } + + public void unsetValue() { + this.value = null; + } + + /** Returns true if field value is set (has been assigned a value) and false otherwise */ + public boolean isSetValue() { + return this.value != null; + } + + public void setValueIsSet(boolean value) { + if (!value) { + this.value = null; + } + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case TXNID: @@ -284,6 +454,46 @@ public void setFieldValue(_Fields field, Object value) { } break; + case CATALOG: + if (value == null) { + unsetCatalog(); + } else { + setCatalog((String)value); + } + break; + + case DATABASE: + if (value == null) { + unsetDatabase(); + } else { + setDatabase((String)value); + } + break; + + case TABLE: + if (value == null) { + unsetTable(); + } else { + setTable((String)value); + } + break; + + case KEY: + if (value == null) { + unsetKey(); + } else { + setKey((String)value); + } + break; + + case VALUE: + if (value == null) { + unsetValue(); + } else { + setValue((String)value); + } + break; + } } @@ -298,6 +508,21 @@ public Object getFieldValue(_Fields field) { case WRITE_EVENT_INFOS: return getWriteEventInfos(); + case CATALOG: + return getCatalog(); + + case DATABASE: + return getDatabase(); + + case TABLE: + return getTable(); + + case KEY: + return getKey(); + + case VALUE: + return getValue(); + } throw new IllegalStateException(); } @@ -315,6 +540,16 @@ public boolean isSet(_Fields field) { return isSetReplPolicy(); case WRITE_EVENT_INFOS: return isSetWriteEventInfos(); + case CATALOG: + return isSetCatalog(); + case DATABASE: + return isSetDatabase(); + case TABLE: + return isSetTable(); + case KEY: + return isSetKey(); + case VALUE: + return isSetValue(); } throw new IllegalStateException(); } @@ -359,6 +594,51 @@ public boolean equals(CommitTxnRequest that) { return false; } + boolean this_present_catalog = true && this.isSetCatalog(); + boolean that_present_catalog = true && that.isSetCatalog(); + if (this_present_catalog || that_present_catalog) { + if (!(this_present_catalog && that_present_catalog)) + return false; + if (!this.catalog.equals(that.catalog)) + return false; + } + + boolean this_present_database = true && this.isSetDatabase(); + boolean that_present_database = true && that.isSetDatabase(); + if (this_present_database || that_present_database) { + if (!(this_present_database && that_present_database)) + return false; + if (!this.database.equals(that.database)) + return false; + } + + boolean this_present_table = true && this.isSetTable(); + boolean that_present_table = true && that.isSetTable(); + if (this_present_table || that_present_table) { + if (!(this_present_table && that_present_table)) + return false; + if (!this.table.equals(that.table)) + return false; + } + + boolean this_present_key = true && this.isSetKey(); + boolean that_present_key = true && that.isSetKey(); + if (this_present_key || that_present_key) { + if (!(this_present_key && that_present_key)) + return false; + if (!this.key.equals(that.key)) + return false; + } + + boolean this_present_value = true && this.isSetValue(); + boolean that_present_value = true && that.isSetValue(); + if (this_present_value || that_present_value) { + if (!(this_present_value && that_present_value)) + return false; + if (!this.value.equals(that.value)) + return false; + } + return true; } @@ -381,6 +661,31 @@ public int hashCode() { if (present_writeEventInfos) list.add(writeEventInfos); + boolean present_catalog = true && (isSetCatalog()); + list.add(present_catalog); + if (present_catalog) + list.add(catalog); + + boolean present_database = true && (isSetDatabase()); + list.add(present_database); + if (present_database) + list.add(database); + + boolean present_table = true && (isSetTable()); + list.add(present_table); + if (present_table) + list.add(table); + + boolean present_key = true && (isSetKey()); + list.add(present_key); + if (present_key) + list.add(key); + + boolean present_value = true && (isSetValue()); + list.add(present_value); + if (present_value) + list.add(value); + return list.hashCode(); } @@ -422,6 +727,56 @@ public int compareTo(CommitTxnRequest other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetCatalog()).compareTo(other.isSetCatalog()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCatalog()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catalog, other.catalog); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetDatabase()).compareTo(other.isSetDatabase()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDatabase()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.database, other.database); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTable()).compareTo(other.isSetTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.table, other.table); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetKey()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -465,6 +820,56 @@ public String toString() { } first = false; } + if (isSetCatalog()) { + if (!first) sb.append(", "); + sb.append("catalog:"); + if (this.catalog == null) { + sb.append("null"); + } else { + sb.append(this.catalog); + } + first = false; + } + if (isSetDatabase()) { + if (!first) sb.append(", "); + sb.append("database:"); + if (this.database == null) { + sb.append("null"); + } else { + sb.append(this.database); + } + first = false; + } + if (isSetTable()) { + if (!first) sb.append(", "); + sb.append("table:"); + if (this.table == null) { + sb.append("null"); + } else { + sb.append(this.table); + } + first = false; + } + if (isSetKey()) { + if (!first) sb.append(", "); + sb.append("key:"); + if (this.key == null) { + sb.append("null"); + } else { + sb.append(this.key); + } + first = false; + } + if (isSetValue()) { + if (!first) sb.append(", "); + sb.append("value:"); + if (this.value == null) { + sb.append("null"); + } else { + sb.append(this.value); + } + first = false; + } sb.append(")"); return sb.toString(); } @@ -549,6 +954,46 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, CommitTxnRequest st org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 4: // CATALOG + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.catalog = iprot.readString(); + struct.setCatalogIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // DATABASE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.database = iprot.readString(); + struct.setDatabaseIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.table = iprot.readString(); + struct.setTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 7: // KEY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.key = iprot.readString(); + struct.setKeyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 8: // VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.value = iprot.readString(); + struct.setValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -586,6 +1031,41 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, CommitTxnRequest s oprot.writeFieldEnd(); } } + if (struct.catalog != null) { + if (struct.isSetCatalog()) { + oprot.writeFieldBegin(CATALOG_FIELD_DESC); + oprot.writeString(struct.catalog); + oprot.writeFieldEnd(); + } + } + if (struct.database != null) { + if (struct.isSetDatabase()) { + oprot.writeFieldBegin(DATABASE_FIELD_DESC); + oprot.writeString(struct.database); + oprot.writeFieldEnd(); + } + } + if (struct.table != null) { + if (struct.isSetTable()) { + oprot.writeFieldBegin(TABLE_FIELD_DESC); + oprot.writeString(struct.table); + oprot.writeFieldEnd(); + } + } + if (struct.key != null) { + if (struct.isSetKey()) { + oprot.writeFieldBegin(KEY_FIELD_DESC); + oprot.writeString(struct.key); + oprot.writeFieldEnd(); + } + } + if (struct.value != null) { + if (struct.isSetValue()) { + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeString(struct.value); + oprot.writeFieldEnd(); + } + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -611,7 +1091,22 @@ public void write(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest st if (struct.isSetWriteEventInfos()) { optionals.set(1); } - oprot.writeBitSet(optionals, 2); + if (struct.isSetCatalog()) { + optionals.set(2); + } + if (struct.isSetDatabase()) { + optionals.set(3); + } + if (struct.isSetTable()) { + optionals.set(4); + } + if (struct.isSetKey()) { + optionals.set(5); + } + if (struct.isSetValue()) { + optionals.set(6); + } + oprot.writeBitSet(optionals, 7); if (struct.isSetReplPolicy()) { oprot.writeString(struct.replPolicy); } @@ -624,6 +1119,21 @@ public void write(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest st } } } + if (struct.isSetCatalog()) { + oprot.writeString(struct.catalog); + } + if (struct.isSetDatabase()) { + oprot.writeString(struct.database); + } + if (struct.isSetTable()) { + oprot.writeString(struct.table); + } + if (struct.isSetKey()) { + oprot.writeString(struct.key); + } + if (struct.isSetValue()) { + oprot.writeString(struct.value); + } } @Override @@ -631,7 +1141,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest str TTupleProtocol iprot = (TTupleProtocol) prot; struct.txnid = iprot.readI64(); struct.setTxnidIsSet(true); - BitSet incoming = iprot.readBitSet(2); + BitSet incoming = iprot.readBitSet(7); if (incoming.get(0)) { struct.replPolicy = iprot.readString(); struct.setReplPolicyIsSet(true); @@ -650,6 +1160,26 @@ public void read(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest str } struct.setWriteEventInfosIsSet(true); } + if (incoming.get(2)) { + struct.catalog = iprot.readString(); + struct.setCatalogIsSet(true); + } + if (incoming.get(3)) { + struct.database = iprot.readString(); + struct.setDatabaseIsSet(true); + } + if (incoming.get(4)) { + struct.table = iprot.readString(); + struct.setTableIsSet(true); + } + if (incoming.get(5)) { + struct.key = iprot.readString(); + struct.setKeyIsSet(true); + } + if (incoming.get(6)) { + struct.value = iprot.readString(); + struct.setValueIsSet(true); + } } } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php index 22deffe1d3..2407ae6a78 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php @@ -17248,6 +17248,26 @@ class CommitTxnRequest { * @var \metastore\WriteEventInfo[] */ public $writeEventInfos = null; + /** + * @var string + */ + public $catalog = null; + /** + * @var string + */ + public $database = null; + /** + * @var string + */ + public $table = null; + /** + * @var string + */ + public $key = null; + /** + * @var string + */ + public $value = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -17269,6 +17289,26 @@ class CommitTxnRequest { 'class' => '\metastore\WriteEventInfo', ), ), + 4 => array( + 'var' => 'catalog', + 'type' => TType::STRING, + ), + 5 => array( + 'var' => 'database', + 'type' => TType::STRING, + ), + 6 => array( + 'var' => 'table', + 'type' => TType::STRING, + ), + 7 => array( + 'var' => 'key', + 'type' => TType::STRING, + ), + 8 => array( + 'var' => 'value', + 'type' => TType::STRING, + ), ); } if (is_array($vals)) { @@ -17281,6 +17321,21 @@ class CommitTxnRequest { if (isset($vals['writeEventInfos'])) { $this->writeEventInfos = $vals['writeEventInfos']; } + if (isset($vals['catalog'])) { + $this->catalog = $vals['catalog']; + } + if (isset($vals['database'])) { + $this->database = $vals['database']; + } + if (isset($vals['table'])) { + $this->table = $vals['table']; + } + if (isset($vals['key'])) { + $this->key = $vals['key']; + } + if (isset($vals['value'])) { + $this->value = $vals['value']; + } } } @@ -17335,6 +17390,41 @@ class CommitTxnRequest { $xfer += $input->skip($ftype); } break; + case 4: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->catalog); + } else { + $xfer += $input->skip($ftype); + } + break; + case 5: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->database); + } else { + $xfer += $input->skip($ftype); + } + break; + case 6: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->table); + } else { + $xfer += $input->skip($ftype); + } + break; + case 7: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->key); + } else { + $xfer += $input->skip($ftype); + } + break; + case 8: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->value); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -17375,6 +17465,31 @@ class CommitTxnRequest { } $xfer += $output->writeFieldEnd(); } + if ($this->catalog !== null) { + $xfer += $output->writeFieldBegin('catalog', TType::STRING, 4); + $xfer += $output->writeString($this->catalog); + $xfer += $output->writeFieldEnd(); + } + if ($this->database !== null) { + $xfer += $output->writeFieldBegin('database', TType::STRING, 5); + $xfer += $output->writeString($this->database); + $xfer += $output->writeFieldEnd(); + } + if ($this->table !== null) { + $xfer += $output->writeFieldBegin('table', TType::STRING, 6); + $xfer += $output->writeString($this->table); + $xfer += $output->writeFieldEnd(); + } + if ($this->key !== null) { + $xfer += $output->writeFieldBegin('key', TType::STRING, 7); + $xfer += $output->writeString($this->key); + $xfer += $output->writeFieldEnd(); + } + if ($this->value !== null) { + $xfer += $output->writeFieldBegin('value', TType::STRING, 8); + $xfer += $output->writeString($this->value); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 38fac465d7..56bfd0d5cc 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -12013,6 +12013,11 @@ class CommitTxnRequest: - txnid - replPolicy - writeEventInfos + - catalog + - database + - table + - key + - value """ thrift_spec = ( @@ -12020,12 +12025,22 @@ class CommitTxnRequest: (1, TType.I64, 'txnid', None, None, ), # 1 (2, TType.STRING, 'replPolicy', None, None, ), # 2 (3, TType.LIST, 'writeEventInfos', (TType.STRUCT,(WriteEventInfo, WriteEventInfo.thrift_spec)), None, ), # 3 + (4, TType.STRING, 'catalog', None, None, ), # 4 + (5, TType.STRING, 'database', None, None, ), # 5 + (6, TType.STRING, 'table', None, None, ), # 6 + (7, TType.STRING, 'key', None, None, ), # 7 + (8, TType.STRING, 'value', None, None, ), # 8 ) - def __init__(self, txnid=None, replPolicy=None, writeEventInfos=None,): + def __init__(self, txnid=None, replPolicy=None, writeEventInfos=None, catalog=None, database=None, table=None, key=None, value=None,): self.txnid = txnid self.replPolicy = replPolicy self.writeEventInfos = writeEventInfos + self.catalog = catalog + self.database = database + self.table = table + self.key = key + self.value = value def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -12057,6 +12072,31 @@ def read(self, iprot): iprot.readListEnd() else: iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.catalog = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.database = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.table = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.key = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.value = iprot.readString() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -12082,6 +12122,26 @@ def write(self, oprot): iter536.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() + if self.catalog is not None: + oprot.writeFieldBegin('catalog', TType.STRING, 4) + oprot.writeString(self.catalog) + oprot.writeFieldEnd() + if self.database is not None: + oprot.writeFieldBegin('database', TType.STRING, 5) + oprot.writeString(self.database) + oprot.writeFieldEnd() + if self.table is not None: + oprot.writeFieldBegin('table', TType.STRING, 6) + oprot.writeString(self.table) + oprot.writeFieldEnd() + if self.key is not None: + oprot.writeFieldBegin('key', TType.STRING, 7) + oprot.writeString(self.key) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin('value', TType.STRING, 8) + oprot.writeString(self.value) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -12096,6 +12156,11 @@ def __hash__(self): value = (value * 31) ^ hash(self.txnid) value = (value * 31) ^ hash(self.replPolicy) value = (value * 31) ^ hash(self.writeEventInfos) + value = (value * 31) ^ hash(self.catalog) + value = (value * 31) ^ hash(self.database) + value = (value * 31) ^ hash(self.table) + value = (value * 31) ^ hash(self.key) + value = (value * 31) ^ hash(self.value) return value def __repr__(self): diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb index 0192c6da31..4218f235b7 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -2673,11 +2673,21 @@ class CommitTxnRequest TXNID = 1 REPLPOLICY = 2 WRITEEVENTINFOS = 3 + CATALOG = 4 + DATABASE = 5 + TABLE = 6 + KEY = 7 + VALUE = 8 FIELDS = { TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid'}, REPLPOLICY => {:type => ::Thrift::Types::STRING, :name => 'replPolicy', :optional => true}, - WRITEEVENTINFOS => {:type => ::Thrift::Types::LIST, :name => 'writeEventInfos', :element => {:type => ::Thrift::Types::STRUCT, :class => ::WriteEventInfo}, :optional => true} + WRITEEVENTINFOS => {:type => ::Thrift::Types::LIST, :name => 'writeEventInfos', :element => {:type => ::Thrift::Types::STRUCT, :class => ::WriteEventInfo}, :optional => true}, + CATALOG => {:type => ::Thrift::Types::STRING, :name => 'catalog', :optional => true}, + DATABASE => {:type => ::Thrift::Types::STRING, :name => 'database', :optional => true}, + TABLE => {:type => ::Thrift::Types::STRING, :name => 'table', :optional => true}, + KEY => {:type => ::Thrift::Types::STRING, :name => 'key', :optional => true}, + VALUE => {:type => ::Thrift::Types::STRING, :name => 'value', :optional => true} } def struct_fields; FIELDS; end diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 85a5c601e0..0f0aa35b87 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -901,6 +901,14 @@ struct CommitTxnRequest { 2: optional string replPolicy, // Information related to write operations done in this transaction. 3: optional list writeEventInfos, + + // An optional key/value to store atomically with the transaction + // If the key and value are set, the catalog, database and table must be set as well + 4: optional string catalog, + 5: optional string database, + 6: optional string table, + 7: optional string key, + 8: optional string value, } struct WriteEventInfo { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index df6d56b679..3f3bce211d 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -50,6 +50,7 @@ import javax.security.auth.login.LoginException; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -2816,6 +2817,29 @@ public void commitTxn(long txnid) client.commit_txn(new CommitTxnRequest(txnid)); } + @Override + public void commitTxnWithKeyValue(long txnid, String catalog, String database, + String table, String key, String value) throws NoSuchTxnException, + TxnAbortedException, TException { + CommitTxnRequest ctr = new CommitTxnRequest(txnid); + Preconditions.checkNotNull(key, "The key to commit together" + + " with the transaction can't be null"); + Preconditions.checkNotNull(value, "The value to commit together" + + " with the transaction can't be null"); + Preconditions.checkNotNull(catalog, "The catalog can't be" + + " null"); + Preconditions.checkNotNull(database, "The database can't be" + + " null"); + Preconditions.checkNotNull(table, "The table can't be null"); + ctr.setCatalog(catalog); + ctr.setDatabase(database); + ctr.setTable(table); + ctr.setKey(key); + ctr.setValue(value); + + client.commit_txn(ctr); + } + @Override public void replCommitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbortedException, TException { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 54e7eda0da..9038508345 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -2925,6 +2925,28 @@ Function getFunction(String catName, String dbName, String funcName) void commitTxn(long txnid) throws NoSuchTxnException, TxnAbortedException, TException; + /** + * Like commitTxn but it will atomically store as well a key and a value. This + * can be useful for example to know if a transaction already been committed. + * TABLE_PARAMS from the metastore must already have a row with the TBL_ID + * corresponding to the table in the parameters and PARAM_KEY the same as + * key in the parameters. + * @param txnid id of transaction to be committed. + * @param database database of the table the key/value will be associated with + * @param table table the key/value will be associated with. + * @param key key to be committed. + * @param value value to be committed. + * @throws NoSuchTxnException if the requested transaction does not exist. + * This can result fro the transaction having timed out and been deleted by + * the compactor. + * @throws TxnAbortedException if the requested transaction has been + * aborted. This can result from the transaction timing out. + * @throws TException + */ + void commitTxnWithKeyValue(long txnid, String catalog, String database, + String table, String key, String value) throws NoSuchTxnException, + TxnAbortedException, TException; + /** * Commit a transaction. This will also unlock any locks associated with * this transaction. diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index d76049eda1..65e6fd8d49 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -1088,6 +1088,38 @@ public void commitTxn(CommitTxnRequest rqst) LOG.debug("Going to execute update <" + s + ">"); stmt.executeUpdate(s); + // update the key/value associated with the transaction if it has been + // set + if (rqst.isSetKey() && rqst.isSetValue()) { + if (!rqst.isSetCatalog() || !rqst.isSetDatabase() || !rqst.isSetTable()) { + throw new SQLException("If key/value are set, the database and" + + " table must be as well"); + } + s = "SELECT TBL_ID FROM DBS, TBLS WHERE" + + " NAME = " + quoteString(rqst.getDatabase()) + + " AND CTLG_NAME = " + quoteString(rqst.getCatalog()) + + " AND DBS.DB_ID = TBLS.DB_ID" + + " AND TBL_NAME = " + quoteString(rqst.getTable()); + LOG.debug("Going to execute select <" + s + ">"); + ResultSet rsTbls = stmt.executeQuery(s); + rsTbls.next(); + long tblId = rsTbls.getLong(1); + if (rsTbls.next()) { + throw new SQLException("Expecting the given parameters" + + " to determine uniquely the table id"); + } + s = "UPDATE TABLE_PARAMS SET" + + " PARAM_VALUE = " + quoteString(rqst.getValue()) + + " WHERE TBL_ID = " + tblId + + " AND PARAM_KEY = " + quoteString(rqst.getKey()); + LOG.debug("Going to execute update <" + s + ">"); + int affectedRows = stmt.executeUpdate(s); + if (affectedRows != 1) { + throw new SQLException("Error updating the key/value in the sql" + + " backend. One row should have been affected"); + } + } + if (transactionalListeners != null) { MetaStoreListenerNotifier.notifyEventWithDirectSql(transactionalListeners, EventMessage.EventType.COMMIT_TXN, new CommitTxnEvent(txnid, null), dbConn, sqlGenerator); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index ce590d0f55..d1f0938fe3 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -50,6 +50,7 @@ import javax.security.auth.login.LoginException; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -2277,6 +2278,29 @@ public void commitTxn(long txnid) client.commit_txn(new CommitTxnRequest(txnid)); } + @Override + public void commitTxnWithKeyValue(long txnid, String catalog, String database, + String table, String key, String value) throws NoSuchTxnException, + TxnAbortedException, TException { + CommitTxnRequest ctr = new CommitTxnRequest(txnid); + Preconditions.checkNotNull(key, "The key to commit together" + + " with the transaction can't be null"); + Preconditions.checkNotNull(value, "The value to commit together" + + " with the transaction can't be null"); + Preconditions.checkNotNull(catalog, "The catalog can't be" + + " null"); + Preconditions.checkNotNull(database, "The database can't be" + + " null"); + Preconditions.checkNotNull(table, "The table can't be null"); + ctr.setCatalog(catalog); + ctr.setDatabase(database); + ctr.setTable(table); + ctr.setKey(key); + ctr.setValue(value); + + client.commit_txn(ctr); + } + @Override public void replCommitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbortedException, TException { diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java index db4dd9ec42..7665062dba 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse; import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.LockState; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.junit.After; @@ -33,6 +34,11 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.List; /** @@ -52,6 +58,7 @@ private final Configuration conf = MetastoreConf.newMetastoreConf(); private IMetaStoreClient client; + private Connection conn; @Test public void testTxns() throws Exception { @@ -83,6 +90,77 @@ public void testOpenTxnNotExcluded() throws Exception { Assert.assertFalse(validTxns.isTxnValid(4)); } + private void cleanUpParamsTable() throws SQLException { + Statement stm = conn.createStatement(); + stm.execute("DELETE FROM TABLE_PARAMS WHERE TBL_ID = 200 AND PARAM_KEY = 'mykey'"); + stm.execute("DELETE FROM TBLS WHERE TBL_ID = 200"); + stm.execute("DELETE FROM DBS WHERE DB_ID = 20"); + } + + @Test + public void testTxNWithKeyValue() throws Exception { + Statement stm = conn.createStatement(); + stm.executeUpdate("INSERT INTO DBS(DB_ID, NAME, CTLG_NAME," + + " DB_LOCATION_URI) VALUES(20, 'mydb', 'hive', '/')"); + stm.executeUpdate("INSERT INTO TBLS(TBL_ID, DB_ID, TBL_NAME," + + " CREATE_TIME, LAST_ACCESS_TIME, RETENTION)" + + " VALUES(200, 20, 'mytable', 10, 10, 10)"); + stm.executeUpdate("INSERT INTO TABLE_PARAMS(TBL_ID, PARAM_KEY)" + + " VALUES(200, 'mykey')"); + + List tids = client.openTxns("me", 1).getTxn_ids(); + Assert.assertEquals(1L, (long) tids.get(0)); + client.commitTxnWithKeyValue(1, "hive", "mydb", + "mytable", "mykey", "myvalue"); + ValidTxnList validTxns = client.getValidTxns(1); + Assert.assertTrue(validTxns.isTxnValid(1)); + + ResultSet rs = stm.executeQuery("SELECT TBL_ID, PARAM_KEY, PARAM_VALUE" + + " FROM TABLE_PARAMS WHERE TBL_ID = 200"); + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getLong(1), 200); + Assert.assertEquals(rs.getString(2), "mykey"); + Assert.assertEquals(rs.getString(3), "myvalue"); + Assert.assertFalse(rs.next()); + cleanUpParamsTable(); + } + + @Test + public void testTxNWithKeyValueNoTableId() throws Exception { + List tids = client.openTxns("me", 1).getTxn_ids(); + Assert.assertEquals(1L, (long) tids.get(0)); + try { + client.commitTxnWithKeyValue(1, "hive", "mydb", "mytable", "mykey", + "myvalue"); + Assert.fail("Should have raised exception"); + } catch (MetaException e) {} + ValidTxnList validTxns = client.getValidTxns(1); + Assert.assertTrue(validTxns.isTxnValid(1)); + cleanUpParamsTable(); + } + + @Test + public void testTxNWithKeyValueParamsNotUpdated() throws Exception { + Statement stm = conn.createStatement(); + stm.executeUpdate("INSERT INTO DBS(DB_ID, NAME, CTLG_NAME," + + " DB_LOCATION_URI) VALUES(20, 'mydb', 'hive', '/')"); + stm.executeUpdate("INSERT INTO TBLS(TBL_ID, DB_ID, TBL_NAME," + + " CREATE_TIME, LAST_ACCESS_TIME, RETENTION)" + + " VALUES(200, 20, 'mytable', 10, 10, 10)"); + + List tids = client.openTxns("me", 1).getTxn_ids(); + Assert.assertEquals(1L, (long) tids.get(0)); + try { + client.commitTxnWithKeyValue(1, "hive", "mydb", "mytable", "mykey", + "myvalue"); + Assert.fail("Exception should have been raised because" + + " no rows were updated"); + } catch (MetaException e) {} + ValidTxnList validTxns = client.getValidTxns(1); + Assert.assertTrue(validTxns.isTxnValid(1)); + cleanUpParamsTable(); + } + @Test public void testTxnRange() throws Exception { ValidTxnList validTxns = client.getValidTxns(); @@ -258,10 +336,14 @@ public void setUp() throws Exception { TxnDbUtil.setConfValues(conf); TxnDbUtil.prepDb(conf); client = new HiveMetaStoreClient(conf); + String connectionStr = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY); + + conn = DriverManager.getConnection(connectionStr); } @After public void tearDown() throws Exception { + conn.close(); TxnDbUtil.cleanDb(conf); } }