diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnKeyValue.java standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnKeyValue.java new file mode 100644 index 0000000000..297cf59a22 --- /dev/null +++ standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnKeyValue.java @@ -0,0 +1,589 @@ +/** + * Autogenerated by Thrift Compiler (0.9.3) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.metastore.api; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.server.AbstractNonblockingServer.*; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import javax.annotation.Generated; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)") +@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class CommitTxnKeyValue implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CommitTxnKeyValue"); + + private static final org.apache.thrift.protocol.TField TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("tableId", org.apache.thrift.protocol.TType.I64, (short)1); + private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new CommitTxnKeyValueStandardSchemeFactory()); + schemes.put(TupleScheme.class, new CommitTxnKeyValueTupleSchemeFactory()); + } + + private long tableId; // required + private String key; // required + private String value; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TABLE_ID((short)1, "tableId"), + KEY((short)2, "key"), + VALUE((short)3, "value"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TABLE_ID + return TABLE_ID; + case 2: // KEY + return KEY; + case 3: // VALUE + return VALUE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TABLEID_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("tableId", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CommitTxnKeyValue.class, metaDataMap); + } + + public CommitTxnKeyValue() { + } + + public CommitTxnKeyValue( + long tableId, + String key, + String value) + { + this(); + this.tableId = tableId; + setTableIdIsSet(true); + this.key = key; + this.value = value; + } + + /** + * Performs a deep copy on other. + */ + public CommitTxnKeyValue(CommitTxnKeyValue other) { + __isset_bitfield = other.__isset_bitfield; + this.tableId = other.tableId; + if (other.isSetKey()) { + this.key = other.key; + } + if (other.isSetValue()) { + this.value = other.value; + } + } + + public CommitTxnKeyValue deepCopy() { + return new CommitTxnKeyValue(this); + } + + @Override + public void clear() { + setTableIdIsSet(false); + this.tableId = 0; + this.key = null; + this.value = null; + } + + public long getTableId() { + return this.tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + setTableIdIsSet(true); + } + + public void unsetTableId() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TABLEID_ISSET_ID); + } + + /** Returns true if field tableId is set (has been assigned a value) and false otherwise */ + public boolean isSetTableId() { + return EncodingUtils.testBit(__isset_bitfield, __TABLEID_ISSET_ID); + } + + public void setTableIdIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TABLEID_ISSET_ID, value); + } + + public String getKey() { + return this.key; + } + + public void setKey(String key) { + this.key = key; + } + + public void unsetKey() { + this.key = null; + } + + /** Returns true if field key is set (has been assigned a value) and false otherwise */ + public boolean isSetKey() { + return this.key != null; + } + + public void setKeyIsSet(boolean value) { + if (!value) { + this.key = null; + } + } + + public String getValue() { + return this.value; + } + + public void setValue(String value) { + this.value = value; + } + + public void unsetValue() { + this.value = null; + } + + /** Returns true if field value is set (has been assigned a value) and false otherwise */ + public boolean isSetValue() { + return this.value != null; + } + + public void setValueIsSet(boolean value) { + if (!value) { + this.value = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case TABLE_ID: + if (value == null) { + unsetTableId(); + } else { + setTableId((Long)value); + } + break; + + case KEY: + if (value == null) { + unsetKey(); + } else { + setKey((String)value); + } + break; + + case VALUE: + if (value == null) { + unsetValue(); + } else { + setValue((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case TABLE_ID: + return getTableId(); + + case KEY: + return getKey(); + + case VALUE: + return getValue(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case TABLE_ID: + return isSetTableId(); + case KEY: + return isSetKey(); + case VALUE: + return isSetValue(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof CommitTxnKeyValue) + return this.equals((CommitTxnKeyValue)that); + return false; + } + + public boolean equals(CommitTxnKeyValue that) { + if (that == null) + return false; + + boolean this_present_tableId = true; + boolean that_present_tableId = true; + if (this_present_tableId || that_present_tableId) { + if (!(this_present_tableId && that_present_tableId)) + return false; + if (this.tableId != that.tableId) + return false; + } + + boolean this_present_key = true && this.isSetKey(); + boolean that_present_key = true && that.isSetKey(); + if (this_present_key || that_present_key) { + if (!(this_present_key && that_present_key)) + return false; + if (!this.key.equals(that.key)) + return false; + } + + boolean this_present_value = true && this.isSetValue(); + boolean that_present_value = true && that.isSetValue(); + if (this_present_value || that_present_value) { + if (!(this_present_value && that_present_value)) + return false; + if (!this.value.equals(that.value)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_tableId = true; + list.add(present_tableId); + if (present_tableId) + list.add(tableId); + + boolean present_key = true && (isSetKey()); + list.add(present_key); + if (present_key) + list.add(key); + + boolean present_value = true && (isSetValue()); + list.add(present_value); + if (present_value) + list.add(value); + + return list.hashCode(); + } + + @Override + public int compareTo(CommitTxnKeyValue other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetTableId()).compareTo(other.isSetTableId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableId, other.tableId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetKey()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetValue()).compareTo(other.isSetValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.value, other.value); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("CommitTxnKeyValue("); + boolean first = true; + + sb.append("tableId:"); + sb.append(this.tableId); + first = false; + if (!first) sb.append(", "); + sb.append("key:"); + if (this.key == null) { + sb.append("null"); + } else { + sb.append(this.key); + } + first = false; + if (!first) sb.append(", "); + sb.append("value:"); + if (this.value == null) { + sb.append("null"); + } else { + sb.append(this.value); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + if (!isSetTableId()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'tableId' is unset! Struct:" + toString()); + } + + if (!isSetKey()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'key' is unset! Struct:" + toString()); + } + + if (!isSetValue()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'value' is unset! Struct:" + toString()); + } + + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class CommitTxnKeyValueStandardSchemeFactory implements SchemeFactory { + public CommitTxnKeyValueStandardScheme getScheme() { + return new CommitTxnKeyValueStandardScheme(); + } + } + + private static class CommitTxnKeyValueStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, CommitTxnKeyValue struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TABLE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.tableId = iprot.readI64(); + struct.setTableIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // KEY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.key = iprot.readString(); + struct.setKeyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.value = iprot.readString(); + struct.setValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, CommitTxnKeyValue struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(TABLE_ID_FIELD_DESC); + oprot.writeI64(struct.tableId); + oprot.writeFieldEnd(); + if (struct.key != null) { + oprot.writeFieldBegin(KEY_FIELD_DESC); + oprot.writeString(struct.key); + oprot.writeFieldEnd(); + } + if (struct.value != null) { + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeString(struct.value); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class CommitTxnKeyValueTupleSchemeFactory implements SchemeFactory { + public CommitTxnKeyValueTupleScheme getScheme() { + return new CommitTxnKeyValueTupleScheme(); + } + } + + private static class CommitTxnKeyValueTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, CommitTxnKeyValue struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeI64(struct.tableId); + oprot.writeString(struct.key); + oprot.writeString(struct.value); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, CommitTxnKeyValue struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + struct.tableId = iprot.readI64(); + struct.setTableIdIsSet(true); + struct.key = iprot.readString(); + struct.setKeyIsSet(true); + struct.value = iprot.readString(); + struct.setValueIsSet(true); + } + } + +} + diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java index db47f9db8b..95ea81f267 100644 --- standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java +++ standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java @@ -41,6 +41,7 @@ private static final org.apache.thrift.protocol.TField TXNID_FIELD_DESC = new org.apache.thrift.protocol.TField("txnid", org.apache.thrift.protocol.TType.I64, (short)1); private static final org.apache.thrift.protocol.TField REPL_POLICY_FIELD_DESC = new org.apache.thrift.protocol.TField("replPolicy", org.apache.thrift.protocol.TType.STRING, (short)2); private static final org.apache.thrift.protocol.TField WRITE_EVENT_INFOS_FIELD_DESC = new org.apache.thrift.protocol.TField("writeEventInfos", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField KEY_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("keyValue", org.apache.thrift.protocol.TType.STRUCT, (short)4); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -51,12 +52,14 @@ private long txnid; // required private String replPolicy; // optional private List writeEventInfos; // optional + private CommitTxnKeyValue keyValue; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { TXNID((short)1, "txnid"), REPL_POLICY((short)2, "replPolicy"), - WRITE_EVENT_INFOS((short)3, "writeEventInfos"); + WRITE_EVENT_INFOS((short)3, "writeEventInfos"), + KEY_VALUE((short)4, "keyValue"); private static final Map byName = new HashMap(); @@ -77,6 +80,8 @@ public static _Fields findByThriftId(int fieldId) { return REPL_POLICY; case 3: // WRITE_EVENT_INFOS return WRITE_EVENT_INFOS; + case 4: // KEY_VALUE + return KEY_VALUE; default: return null; } @@ -119,7 +124,7 @@ public String getFieldName() { // isset id assignments private static final int __TXNID_ISSET_ID = 0; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.REPL_POLICY,_Fields.WRITE_EVENT_INFOS}; + private static final _Fields optionals[] = {_Fields.REPL_POLICY,_Fields.WRITE_EVENT_INFOS,_Fields.KEY_VALUE}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -130,6 +135,8 @@ public String getFieldName() { tmpMap.put(_Fields.WRITE_EVENT_INFOS, new org.apache.thrift.meta_data.FieldMetaData("writeEventInfos", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT , "WriteEventInfo")))); + tmpMap.put(_Fields.KEY_VALUE, new org.apache.thrift.meta_data.FieldMetaData("keyValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CommitTxnKeyValue.class))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CommitTxnRequest.class, metaDataMap); } @@ -161,6 +168,9 @@ public CommitTxnRequest(CommitTxnRequest other) { } this.writeEventInfos = __this__writeEventInfos; } + if (other.isSetKeyValue()) { + this.keyValue = new CommitTxnKeyValue(other.keyValue); + } } public CommitTxnRequest deepCopy() { @@ -173,6 +183,7 @@ public void clear() { this.txnid = 0; this.replPolicy = null; this.writeEventInfos = null; + this.keyValue = null; } public long getTxnid() { @@ -258,6 +269,29 @@ public void setWriteEventInfosIsSet(boolean value) { } } + public CommitTxnKeyValue getKeyValue() { + return this.keyValue; + } + + public void setKeyValue(CommitTxnKeyValue keyValue) { + this.keyValue = keyValue; + } + + public void unsetKeyValue() { + this.keyValue = null; + } + + /** Returns true if field keyValue is set (has been assigned a value) and false otherwise */ + public boolean isSetKeyValue() { + return this.keyValue != null; + } + + public void setKeyValueIsSet(boolean value) { + if (!value) { + this.keyValue = null; + } + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case TXNID: @@ -284,6 +318,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case KEY_VALUE: + if (value == null) { + unsetKeyValue(); + } else { + setKeyValue((CommitTxnKeyValue)value); + } + break; + } } @@ -298,6 +340,9 @@ public Object getFieldValue(_Fields field) { case WRITE_EVENT_INFOS: return getWriteEventInfos(); + case KEY_VALUE: + return getKeyValue(); + } throw new IllegalStateException(); } @@ -315,6 +360,8 @@ public boolean isSet(_Fields field) { return isSetReplPolicy(); case WRITE_EVENT_INFOS: return isSetWriteEventInfos(); + case KEY_VALUE: + return isSetKeyValue(); } throw new IllegalStateException(); } @@ -359,6 +406,15 @@ public boolean equals(CommitTxnRequest that) { return false; } + boolean this_present_keyValue = true && this.isSetKeyValue(); + boolean that_present_keyValue = true && that.isSetKeyValue(); + if (this_present_keyValue || that_present_keyValue) { + if (!(this_present_keyValue && that_present_keyValue)) + return false; + if (!this.keyValue.equals(that.keyValue)) + return false; + } + return true; } @@ -381,6 +437,11 @@ public int hashCode() { if (present_writeEventInfos) list.add(writeEventInfos); + boolean present_keyValue = true && (isSetKeyValue()); + list.add(present_keyValue); + if (present_keyValue) + list.add(keyValue); + return list.hashCode(); } @@ -422,6 +483,16 @@ public int compareTo(CommitTxnRequest other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetKeyValue()).compareTo(other.isSetKeyValue()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetKeyValue()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyValue, other.keyValue); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -465,6 +536,16 @@ public String toString() { } first = false; } + if (isSetKeyValue()) { + if (!first) sb.append(", "); + sb.append("keyValue:"); + if (this.keyValue == null) { + sb.append("null"); + } else { + sb.append(this.keyValue); + } + first = false; + } sb.append(")"); return sb.toString(); } @@ -476,6 +557,9 @@ public void validate() throws org.apache.thrift.TException { } // check for sub-struct validity + if (keyValue != null) { + keyValue.validate(); + } } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { @@ -549,6 +633,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, CommitTxnRequest st org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 4: // KEY_VALUE + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.keyValue = new CommitTxnKeyValue(); + struct.keyValue.read(iprot); + struct.setKeyValueIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -586,6 +679,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, CommitTxnRequest s oprot.writeFieldEnd(); } } + if (struct.keyValue != null) { + if (struct.isSetKeyValue()) { + oprot.writeFieldBegin(KEY_VALUE_FIELD_DESC); + struct.keyValue.write(oprot); + oprot.writeFieldEnd(); + } + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -611,7 +711,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest st if (struct.isSetWriteEventInfos()) { optionals.set(1); } - oprot.writeBitSet(optionals, 2); + if (struct.isSetKeyValue()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); if (struct.isSetReplPolicy()) { oprot.writeString(struct.replPolicy); } @@ -624,6 +727,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest st } } } + if (struct.isSetKeyValue()) { + struct.keyValue.write(oprot); + } } @Override @@ -631,7 +737,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest str TTupleProtocol iprot = (TTupleProtocol) prot; struct.txnid = iprot.readI64(); struct.setTxnidIsSet(true); - BitSet incoming = iprot.readBitSet(2); + BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { struct.replPolicy = iprot.readString(); struct.setReplPolicyIsSet(true); @@ -650,6 +756,11 @@ public void read(org.apache.thrift.protocol.TProtocol prot, CommitTxnRequest str } struct.setWriteEventInfosIsSet(true); } + if (incoming.get(2)) { + struct.keyValue = new CommitTxnKeyValue(); + struct.keyValue.read(iprot); + struct.setKeyValueIsSet(true); + } } } diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php index 936f7c5a40..b5a943ee4f 100644 --- standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php +++ standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php @@ -17256,6 +17256,127 @@ class AbortTxnsRequest { } +class CommitTxnKeyValue { + static $_TSPEC; + + /** + * @var int + */ + public $tableId = null; + /** + * @var string + */ + public $key = null; + /** + * @var string + */ + public $value = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 1 => array( + 'var' => 'tableId', + 'type' => TType::I64, + ), + 2 => array( + 'var' => 'key', + 'type' => TType::STRING, + ), + 3 => array( + 'var' => 'value', + 'type' => TType::STRING, + ), + ); + } + if (is_array($vals)) { + if (isset($vals['tableId'])) { + $this->tableId = $vals['tableId']; + } + if (isset($vals['key'])) { + $this->key = $vals['key']; + } + if (isset($vals['value'])) { + $this->value = $vals['value']; + } + } + } + + public function getName() { + return 'CommitTxnKeyValue'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 1: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->tableId); + } else { + $xfer += $input->skip($ftype); + } + break; + case 2: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->key); + } else { + $xfer += $input->skip($ftype); + } + break; + case 3: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->value); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('CommitTxnKeyValue'); + if ($this->tableId !== null) { + $xfer += $output->writeFieldBegin('tableId', TType::I64, 1); + $xfer += $output->writeI64($this->tableId); + $xfer += $output->writeFieldEnd(); + } + if ($this->key !== null) { + $xfer += $output->writeFieldBegin('key', TType::STRING, 2); + $xfer += $output->writeString($this->key); + $xfer += $output->writeFieldEnd(); + } + if ($this->value !== null) { + $xfer += $output->writeFieldBegin('value', TType::STRING, 3); + $xfer += $output->writeString($this->value); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + class CommitTxnRequest { static $_TSPEC; @@ -17271,6 +17392,10 @@ class CommitTxnRequest { * @var \metastore\WriteEventInfo[] */ public $writeEventInfos = null; + /** + * @var \metastore\CommitTxnKeyValue + */ + public $keyValue = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -17292,6 +17417,11 @@ class CommitTxnRequest { 'class' => '\metastore\WriteEventInfo', ), ), + 4 => array( + 'var' => 'keyValue', + 'type' => TType::STRUCT, + 'class' => '\metastore\CommitTxnKeyValue', + ), ); } if (is_array($vals)) { @@ -17304,6 +17434,9 @@ class CommitTxnRequest { if (isset($vals['writeEventInfos'])) { $this->writeEventInfos = $vals['writeEventInfos']; } + if (isset($vals['keyValue'])) { + $this->keyValue = $vals['keyValue']; + } } } @@ -17358,6 +17491,14 @@ class CommitTxnRequest { $xfer += $input->skip($ftype); } break; + case 4: + if ($ftype == TType::STRUCT) { + $this->keyValue = new \metastore\CommitTxnKeyValue(); + $xfer += $this->keyValue->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -17398,6 +17539,14 @@ class CommitTxnRequest { } $xfer += $output->writeFieldEnd(); } + if ($this->keyValue !== null) { + if (!is_object($this->keyValue)) { + throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA); + } + $xfer += $output->writeFieldBegin('keyValue', TType::STRUCT, 4); + $xfer += $this->keyValue->write($output); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 958f13c18e..c28976a8e1 100644 --- standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -12020,12 +12020,110 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) +class CommitTxnKeyValue: + """ + Attributes: + - tableId + - key + - value + """ + + thrift_spec = ( + None, # 0 + (1, TType.I64, 'tableId', None, None, ), # 1 + (2, TType.STRING, 'key', None, None, ), # 2 + (3, TType.STRING, 'value', None, None, ), # 3 + ) + + def __init__(self, tableId=None, key=None, value=None,): + self.tableId = tableId + self.key = key + self.value = value + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.key = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.value = iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('CommitTxnKeyValue') + if self.tableId is not None: + oprot.writeFieldBegin('tableId', TType.I64, 1) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + if self.key is not None: + oprot.writeFieldBegin('key', TType.STRING, 2) + oprot.writeString(self.key) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin('value', TType.STRING, 3) + oprot.writeString(self.value) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.tableId is None: + raise TProtocol.TProtocolException(message='Required field tableId is unset!') + if self.key is None: + raise TProtocol.TProtocolException(message='Required field key is unset!') + if self.value is None: + raise TProtocol.TProtocolException(message='Required field value is unset!') + return + + + def __hash__(self): + value = 17 + value = (value * 31) ^ hash(self.tableId) + value = (value * 31) ^ hash(self.key) + value = (value * 31) ^ hash(self.value) + return value + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + class CommitTxnRequest: """ Attributes: - txnid - replPolicy - writeEventInfos + - keyValue """ thrift_spec = ( @@ -12033,12 +12131,14 @@ class CommitTxnRequest: (1, TType.I64, 'txnid', None, None, ), # 1 (2, TType.STRING, 'replPolicy', None, None, ), # 2 (3, TType.LIST, 'writeEventInfos', (TType.STRUCT,(WriteEventInfo, WriteEventInfo.thrift_spec)), None, ), # 3 + (4, TType.STRUCT, 'keyValue', (CommitTxnKeyValue, CommitTxnKeyValue.thrift_spec), None, ), # 4 ) - def __init__(self, txnid=None, replPolicy=None, writeEventInfos=None,): + def __init__(self, txnid=None, replPolicy=None, writeEventInfos=None, keyValue=None,): self.txnid = txnid self.replPolicy = replPolicy self.writeEventInfos = writeEventInfos + self.keyValue = keyValue def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -12070,6 +12170,12 @@ def read(self, iprot): iprot.readListEnd() else: iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.keyValue = CommitTxnKeyValue() + self.keyValue.read(iprot) + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -12095,6 +12201,10 @@ def write(self, oprot): iter536.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() + if self.keyValue is not None: + oprot.writeFieldBegin('keyValue', TType.STRUCT, 4) + self.keyValue.write(oprot) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -12109,6 +12219,7 @@ def __hash__(self): value = (value * 31) ^ hash(self.txnid) value = (value * 31) ^ hash(self.replPolicy) value = (value * 31) ^ hash(self.writeEventInfos) + value = (value * 31) ^ hash(self.keyValue) return value def __repr__(self): diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb index a3dddf54e4..04c04513a4 100644 --- standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -2670,16 +2670,41 @@ class AbortTxnsRequest ::Thrift::Struct.generate_accessors self end +class CommitTxnKeyValue + include ::Thrift::Struct, ::Thrift::Struct_Union + TABLEID = 1 + KEY = 2 + VALUE = 3 + + FIELDS = { + TABLEID => {:type => ::Thrift::Types::I64, :name => 'tableId'}, + KEY => {:type => ::Thrift::Types::STRING, :name => 'key'}, + VALUE => {:type => ::Thrift::Types::STRING, :name => 'value'} + } + + def struct_fields; FIELDS; end + + def validate + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tableId is unset!') unless @tableId + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field key is unset!') unless @key + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field value is unset!') unless @value + end + + ::Thrift::Struct.generate_accessors self +end + class CommitTxnRequest include ::Thrift::Struct, ::Thrift::Struct_Union TXNID = 1 REPLPOLICY = 2 WRITEEVENTINFOS = 3 + KEYVALUE = 4 FIELDS = { TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid'}, REPLPOLICY => {:type => ::Thrift::Types::STRING, :name => 'replPolicy', :optional => true}, - WRITEEVENTINFOS => {:type => ::Thrift::Types::LIST, :name => 'writeEventInfos', :element => {:type => ::Thrift::Types::STRUCT, :class => ::WriteEventInfo}, :optional => true} + WRITEEVENTINFOS => {:type => ::Thrift::Types::LIST, :name => 'writeEventInfos', :element => {:type => ::Thrift::Types::STRUCT, :class => ::WriteEventInfo}, :optional => true}, + KEYVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'keyValue', :class => ::CommitTxnKeyValue, :optional => true} } def struct_fields; FIELDS; end diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index d226db50a5..3fbe8b4965 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -50,6 +50,7 @@ import javax.security.auth.login.LoginException; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -2829,6 +2830,20 @@ public void commitTxn(long txnid) client.commit_txn(new CommitTxnRequest(txnid)); } + @Override + public void commitTxnWithKeyValue(long txnid, long tableId, String key, + String value) throws NoSuchTxnException, + TxnAbortedException, TException { + CommitTxnRequest ctr = new CommitTxnRequest(txnid); + Preconditions.checkNotNull(key, "The key to commit together" + + " with the transaction can't be null"); + Preconditions.checkNotNull(value, "The value to commit together" + + " with the transaction can't be null"); + ctr.setKeyValue(new CommitTxnKeyValue(tableId, key, value)); + + client.commit_txn(ctr); + } + @Override public void replCommitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbortedException, TException { diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 54e7eda0da..660e770277 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -2925,6 +2925,32 @@ Function getFunction(String catName, String dbName, String funcName) void commitTxn(long txnid) throws NoSuchTxnException, TxnAbortedException, TException; + /** + * Like commitTxn but it will atomically store as well a key and a value. This + * can be useful for example to know if the transaction corresponding to + * txnid has been committed by later querying with DESCRIBE EXTENDED TABLE. + * TABLE_PARAMS from the metastore must already have a row with the TBL_ID + * corresponding to the table in the parameters and PARAM_KEY the same as key + * in the parameters. The way to update this table is with an ALTER command + * to overwrite/create the table properties. + * @param txnid id of transaction to be committed. + * @param tableId id of the table to associate the key/value with + * @param key key to be committed. It must start with "_meta". The reason + * for this is to prevent important keys being updated, like owner. + * @param value value to be committed. + * @throws NoSuchTxnException if the requested transaction does not exist. + * This can result fro the transaction having timed out and been deleted by + * the compactor. + * @throws TxnAbortedException if the requested transaction has been + * aborted. This can result from the transaction timing out. + * @throws IllegalStateException if not exactly one row corresponding to + * tableId and key are found in TABLE_PARAMS while updating. + * @throws TException + */ + void commitTxnWithKeyValue(long txnid, long tableId, + String key, String value) throws NoSuchTxnException, + TxnAbortedException, TException; + /** * Commit a transaction. This will also unlock any locks associated with * this transaction. diff --git standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index ad83162ec3..407591091c 100644 --- standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -898,11 +898,20 @@ struct AbortTxnsRequest { 1: required list txn_ids, } +struct CommitTxnKeyValue { + 1: required i64 tableId, + 2: required string key, + 3: required string value, +} + struct CommitTxnRequest { 1: required i64 txnid, 2: optional string replPolicy, // Information related to write operations done in this transaction. 3: optional list writeEventInfos, + + // An optional key/value to store atomically with the transaction + 4: optional CommitTxnKeyValue keyValue, } struct WriteEventInfo { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 1df1ebce49..351fafd933 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -1111,6 +1111,38 @@ public void commitTxn(CommitTxnRequest rqst) LOG.debug("Going to execute update <" + s + ">"); stmt.executeUpdate(s); + // update the key/value associated with the transaction if it has been + // set + if (rqst.isSetKeyValue()) { + if (!rqst.getKeyValue().getKey().startsWith(TxnStore.TXN_KEY_START)) { + String errorMsg = "Error updating key/value in the sql backend with" + + " txnId=" + rqst.getTxnid() + "," + + " tableId=" + rqst.getKeyValue().getTableId() + "," + + " key=" + rqst.getKeyValue().getKey() + "," + + " value=" + rqst.getKeyValue().getValue() + "." + + " key should start with " + TXN_KEY_START + "."; + LOG.warn(errorMsg); + throw new IllegalArgumentException(errorMsg); + } + s = "UPDATE TABLE_PARAMS SET" + + " PARAM_VALUE = " + quoteString(rqst.getKeyValue().getValue()) + + " WHERE TBL_ID = " + rqst.getKeyValue().getTableId() + + " AND PARAM_KEY = " + quoteString(rqst.getKeyValue().getKey()); + LOG.debug("Going to execute update <" + s + ">"); + int affectedRows = stmt.executeUpdate(s); + if (affectedRows != 1) { + String errorMsg = "Error updating key/value in the sql backend with" + + " txnId=" + rqst.getTxnid() + "," + + " tableId=" + rqst.getKeyValue().getTableId() + "," + + " key=" + rqst.getKeyValue().getKey() + "," + + " value=" + rqst.getKeyValue().getValue() + "." + + " Only one row should have been affected but " + + affectedRows + " rows where affected."; + LOG.warn(errorMsg); + throw new IllegalStateException(errorMsg); + } + } + if (transactionalListeners != null) { MetaStoreListenerNotifier.notifyEventWithDirectSql(transactionalListeners, EventMessage.EventType.COMMIT_TXN, new CommitTxnEvent(txnid, null), dbConn, sqlGenerator); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java index 080cc5284b..8d43eca7d4 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java @@ -41,6 +41,10 @@ @InterfaceAudience.Private @InterfaceStability.Evolving public interface TxnStore extends Configurable { + /** + * Prefix for key when committing with a key/value + */ + public static final String TXN_KEY_START = "_meta"; enum MUTEX_KEY { Initiator, Cleaner, HouseKeeper, CompactionHistory, CheckLock, diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index ce590d0f55..3ffd6b038f 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -50,6 +50,7 @@ import javax.security.auth.login.LoginException; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -2277,6 +2278,19 @@ public void commitTxn(long txnid) client.commit_txn(new CommitTxnRequest(txnid)); } + @Override + public void commitTxnWithKeyValue(long txnid, long tableId, String key, + String value) throws NoSuchTxnException, + TxnAbortedException, TException { + CommitTxnRequest ctr = new CommitTxnRequest(txnid); + Preconditions.checkNotNull(key, "The key to commit together" + + " with the transaction can't be null"); + Preconditions.checkNotNull(value, "The value to commit together" + + " with the transaction can't be null"); + ctr.setKeyValue(new CommitTxnKeyValue(tableId, key, value)); + client.commit_txn(ctr); + } + @Override public void replCommitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbortedException, TException { diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java index db4dd9ec42..130b1e8436 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java @@ -22,17 +22,28 @@ import org.apache.hadoop.hive.common.ValidReadTxnList; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.api.DataOperationType; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse; import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.LockState; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; +import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.List; /** @@ -52,6 +63,7 @@ private final Configuration conf = MetastoreConf.newMetastoreConf(); private IMetaStoreClient client; + private Connection conn; @Test public void testTxns() throws Exception { @@ -83,6 +95,96 @@ public void testOpenTxnNotExcluded() throws Exception { Assert.assertFalse(validTxns.isTxnValid(4)); } + @Test + public void testTxNWithKeyValue() throws Exception { + Statement stm = conn.createStatement(); + + String dbName = "mydbKeyValue"; + String tblName = "mytable"; + + Database db = new DatabaseBuilder().setName(dbName).build(conf); + db.unsetCatalogName(); + Table tbl = new TableBuilder().setDbName(dbName).setTableName(tblName) + .addCol("id", "int").addCol("name", "string") + .setType(TableType.MANAGED_TABLE.name()).build(conf); + + try { + client.createDatabase(db); + client.createTable(tbl); + tbl = client.getTable(dbName, tblName); + + stm.executeUpdate( + "INSERT INTO TABLE_PARAMS(TBL_ID, PARAM_KEY)" + " VALUES(" + tbl.getId() + ", TxnStore.TXN_KEY_START + 'mykey')"); + + List tids = client.openTxns("me", 1).getTxn_ids(); + Assert.assertEquals(1L, (long) tids.get(0)); + client.commitTxnWithKeyValue(1, tbl.getId(), TxnStore.TXN_KEY_START + "mykey", "myvalue"); + ValidTxnList validTxns = client.getValidTxns(1); + Assert.assertTrue(validTxns.isTxnValid(1)); + + ResultSet rs = stm.executeQuery("SELECT TBL_ID, PARAM_KEY, PARAM_VALUE" + + " FROM TABLE_PARAMS WHERE TBL_ID = " + tbl.getId()); + + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getLong(1), tbl.getId()); + Assert.assertEquals(rs.getString(2), TxnStore.TXN_KEY_START + "mykey"); + Assert.assertEquals(rs.getString(3), "myvalue"); + } finally { + client.dropTable(dbName, tblName); + client.dropDatabase(dbName); + stm.execute("DELETE FROM TABLE_PARAMS WHERE TBL_ID = " + tbl.getId() + " AND PARAM_KEY = + TxnStore.TXN_KEY_START + 'mykey'"); + } + } + + @Test + public void testTxNWithKeyValueNoTableId() throws Exception { + List tids = client.openTxns("me", 1).getTxn_ids(); + Assert.assertEquals(1L, (long) tids.get(0)); + try { + client.commitTxnWithKeyValue(1, 10, TxnStore.TXN_KEY_START + "mykey", + "myvalue"); + Assert.fail("Should have raised exception"); + } catch (IllegalStateException e) { + Assert.assertTrue(e.getMessage().contains("key=" + TxnStore.TXN_KEY_START + "mykey")); + Assert.assertTrue(e.getMessage().contains("value=myvalue")); + Assert.assertTrue(e.getMessage().contains("Only one row should have been affected but")); + } + ValidTxnList validTxns = client.getValidTxns(1); + Assert.assertTrue(validTxns.isTxnValid(1)); + } + + @Test + public void testTxNWithKeyWrongPrefix() throws Exception { + String dbName = "mydbKeyValueWrongPrefix"; + String tblName = "mytable"; + List tids = client.openTxns("me", 1).getTxn_ids(); + Assert.assertEquals(1L, (long) tids.get(0)); + try { + Database db = new DatabaseBuilder().setName(dbName).build(conf); + db.unsetCatalogName(); + client.createDatabase(db); + + Table tbl = new TableBuilder().setDbName(dbName).setTableName(tblName) + .addCol("id", "int").addCol("name", "string") + .setType(TableType.MANAGED_TABLE.name()).build(conf); + client.createTable(tbl); + tbl = client.getTable(dbName, tblName); + + client.commitTxnWithKeyValue(1, tbl.getId(), "mykey", + "myvalue"); + Assert.fail("Should have raised exception"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("key=mykey")); + Assert.assertTrue(e.getMessage().contains("value=myvalue")); + Assert.assertTrue(e.getMessage().contains("key should start with")); + } finally { + client.dropTable(dbName, tblName); + client.dropDatabase(dbName); + } + ValidTxnList validTxns = client.getValidTxns(1); + Assert.assertTrue(validTxns.isTxnValid(1)); + } + @Test public void testTxnRange() throws Exception { ValidTxnList validTxns = client.getValidTxns(); @@ -258,10 +360,14 @@ public void setUp() throws Exception { TxnDbUtil.setConfValues(conf); TxnDbUtil.prepDb(conf); client = new HiveMetaStoreClient(conf); + String connectionStr = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY); + + conn = DriverManager.getConnection(connectionStr); } @After public void tearDown() throws Exception { + conn.close(); TxnDbUtil.cleanDb(conf); } }