commit 55732f3c1f25de0d34aa2f2476c643988472bb4a Author: Bharath Krishna Date: Tue Jan 22 21:47:21 2019 -0800 HIVE-21115 : Add support for object versions in metastore diff --git metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql index 1fa652da88c9bba122e611e03ae5ed90857a6c27..c5323a173a36fcd280678c6cd6c8d16d8682a8ef 100644 --- metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql +++ metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql @@ -167,6 +167,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITIONS` ( `PART_NAME` string, `SD_ID` bigint, `TBL_ID` bigint, + `VERSION` bigint, CONSTRAINT `SYS_PK_PARTITIONS` PRIMARY KEY (`PART_ID`) DISABLE ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' @@ -593,6 +594,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS `TBLS` ( `VIEW_EXPANDED_TEXT` string, `VIEW_ORIGINAL_TEXT` string, `IS_REWRITE_ENABLED` boolean, + `VERSION` bigint, CONSTRAINT `SYS_PK_TBLS` PRIMARY KEY (`TBL_ID`) DISABLE ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java index 0156a9a39fcb1c6d281f9953af7713e87d017377..54c17249d30ff2e621b1745d250d87d1c1d8c85a 100644 --- standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java +++ standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java @@ -49,6 +49,7 @@ private static final org.apache.thrift.protocol.TField CAT_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catName", org.apache.thrift.protocol.TType.STRING, (short)9); private static final org.apache.thrift.protocol.TField WRITE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("writeId", org.apache.thrift.protocol.TType.I64, (short)10); private static final org.apache.thrift.protocol.TField IS_STATS_COMPLIANT_FIELD_DESC = new org.apache.thrift.protocol.TField("isStatsCompliant", org.apache.thrift.protocol.TType.BOOL, (short)11); + private static final org.apache.thrift.protocol.TField VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("version", org.apache.thrift.protocol.TType.I64, (short)12); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -67,6 +68,7 @@ private String catName; // optional private long writeId; // optional private boolean isStatsCompliant; // optional + private long version; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -80,7 +82,8 @@ PRIVILEGES((short)8, "privileges"), CAT_NAME((short)9, "catName"), WRITE_ID((short)10, "writeId"), - IS_STATS_COMPLIANT((short)11, "isStatsCompliant"); + IS_STATS_COMPLIANT((short)11, "isStatsCompliant"), + VERSION((short)12, "version"); private static final Map byName = new HashMap(); @@ -117,6 +120,8 @@ public static _Fields findByThriftId(int fieldId) { return WRITE_ID; case 11: // IS_STATS_COMPLIANT return IS_STATS_COMPLIANT; + case 12: // VERSION + return VERSION; default: return null; } @@ -161,8 +166,9 @@ public String getFieldName() { private static final int __LASTACCESSTIME_ISSET_ID = 1; private static final int __WRITEID_ISSET_ID = 2; private static final int __ISSTATSCOMPLIANT_ISSET_ID = 3; + private static final int __VERSION_ISSET_ID = 4; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.PRIVILEGES,_Fields.CAT_NAME,_Fields.WRITE_ID,_Fields.IS_STATS_COMPLIANT}; + private static final _Fields optionals[] = {_Fields.PRIVILEGES,_Fields.CAT_NAME,_Fields.WRITE_ID,_Fields.IS_STATS_COMPLIANT,_Fields.VERSION}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -191,6 +197,8 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); tmpMap.put(_Fields.IS_STATS_COMPLIANT, new org.apache.thrift.meta_data.FieldMetaData("isStatsCompliant", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.VERSION, new org.apache.thrift.meta_data.FieldMetaData("version", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Partition.class, metaDataMap); } @@ -198,6 +206,8 @@ public String getFieldName() { public Partition() { this.writeId = -1L; + this.version = 0L; + } public Partition( @@ -253,6 +263,7 @@ public Partition(Partition other) { } this.writeId = other.writeId; this.isStatsCompliant = other.isStatsCompliant; + this.version = other.version; } public Partition deepCopy() { @@ -276,6 +287,8 @@ public void clear() { setIsStatsCompliantIsSet(false); this.isStatsCompliant = false; + this.version = 0L; + } public int getValuesSize() { @@ -553,6 +566,28 @@ public void setIsStatsCompliantIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __ISSTATSCOMPLIANT_ISSET_ID, value); } + public long getVersion() { + return this.version; + } + + public void setVersion(long version) { + this.version = version; + setVersionIsSet(true); + } + + public void unsetVersion() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VERSION_ISSET_ID); + } + + /** Returns true if field version is set (has been assigned a value) and false otherwise */ + public boolean isSetVersion() { + return EncodingUtils.testBit(__isset_bitfield, __VERSION_ISSET_ID); + } + + public void setVersionIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VERSION_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case VALUES: @@ -643,6 +678,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case VERSION: + if (value == null) { + unsetVersion(); + } else { + setVersion((Long)value); + } + break; + } } @@ -681,6 +724,9 @@ public Object getFieldValue(_Fields field) { case IS_STATS_COMPLIANT: return isIsStatsCompliant(); + case VERSION: + return getVersion(); + } throw new IllegalStateException(); } @@ -714,6 +760,8 @@ public boolean isSet(_Fields field) { return isSetWriteId(); case IS_STATS_COMPLIANT: return isSetIsStatsCompliant(); + case VERSION: + return isSetVersion(); } throw new IllegalStateException(); } @@ -830,6 +878,15 @@ public boolean equals(Partition that) { return false; } + boolean this_present_version = true && this.isSetVersion(); + boolean that_present_version = true && that.isSetVersion(); + if (this_present_version || that_present_version) { + if (!(this_present_version && that_present_version)) + return false; + if (this.version != that.version) + return false; + } + return true; } @@ -892,6 +949,11 @@ public int hashCode() { if (present_isStatsCompliant) list.add(isStatsCompliant); + boolean present_version = true && (isSetVersion()); + list.add(present_version); + if (present_version) + list.add(version); + return list.hashCode(); } @@ -1013,6 +1075,16 @@ public int compareTo(Partition other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetVersion()).compareTo(other.isSetVersion()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetVersion()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.version, other.version); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -1112,6 +1184,12 @@ public String toString() { sb.append(this.isStatsCompliant); first = false; } + if (isSetVersion()) { + if (!first) sb.append(", "); + sb.append("version:"); + sb.append(this.version); + first = false; + } sb.append(")"); return sb.toString(); } @@ -1275,6 +1353,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Partition struct) t org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 12: // VERSION + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.version = iprot.readI64(); + struct.setVersionIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1358,6 +1444,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, Partition struct) oprot.writeBool(struct.isStatsCompliant); oprot.writeFieldEnd(); } + if (struct.isSetVersion()) { + oprot.writeFieldBegin(VERSION_FIELD_DESC); + oprot.writeI64(struct.version); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1409,7 +1500,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Partition struct) t if (struct.isSetIsStatsCompliant()) { optionals.set(10); } - oprot.writeBitSet(optionals, 11); + if (struct.isSetVersion()) { + optionals.set(11); + } + oprot.writeBitSet(optionals, 12); if (struct.isSetValues()) { { oprot.writeI32(struct.values.size()); @@ -1456,12 +1550,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Partition struct) t if (struct.isSetIsStatsCompliant()) { oprot.writeBool(struct.isStatsCompliant); } + if (struct.isSetVersion()) { + oprot.writeI64(struct.version); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, Partition struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(11); + BitSet incoming = iprot.readBitSet(12); if (incoming.get(0)) { { org.apache.thrift.protocol.TList _list235 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); @@ -1528,6 +1625,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Partition struct) th struct.isStatsCompliant = iprot.readBool(); struct.setIsStatsCompliantIsSet(true); } + if (incoming.get(11)) { + struct.version = iprot.readI64(); + struct.setVersionIsSet(true); + } } } diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java index 75edb63d1c8fa0d6a14beea5c2813da7017b3adc..51579a858f193ff57965722b6f18324c03fbf09a 100644 --- standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java +++ standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java @@ -60,6 +60,7 @@ private static final org.apache.thrift.protocol.TField WRITE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("writeId", org.apache.thrift.protocol.TType.I64, (short)20); private static final org.apache.thrift.protocol.TField IS_STATS_COMPLIANT_FIELD_DESC = new org.apache.thrift.protocol.TField("isStatsCompliant", org.apache.thrift.protocol.TType.BOOL, (short)21); private static final org.apache.thrift.protocol.TField COL_STATS_FIELD_DESC = new org.apache.thrift.protocol.TField("colStats", org.apache.thrift.protocol.TType.STRUCT, (short)22); + private static final org.apache.thrift.protocol.TField VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("version", org.apache.thrift.protocol.TType.I64, (short)23); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -89,6 +90,7 @@ private long writeId; // optional private boolean isStatsCompliant; // optional private ColumnStatistics colStats; // optional + private long version; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -117,7 +119,8 @@ OWNER_TYPE((short)19, "ownerType"), WRITE_ID((short)20, "writeId"), IS_STATS_COMPLIANT((short)21, "isStatsCompliant"), - COL_STATS((short)22, "colStats"); + COL_STATS((short)22, "colStats"), + VERSION((short)23, "version"); private static final Map byName = new HashMap(); @@ -176,6 +179,8 @@ public static _Fields findByThriftId(int fieldId) { return IS_STATS_COMPLIANT; case 22: // COL_STATS return COL_STATS; + case 23: // VERSION + return VERSION; default: return null; } @@ -224,8 +229,9 @@ public String getFieldName() { private static final int __REWRITEENABLED_ISSET_ID = 5; private static final int __WRITEID_ISSET_ID = 6; private static final int __ISSTATSCOMPLIANT_ISSET_ID = 7; - private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.ID,_Fields.PRIVILEGES,_Fields.TEMPORARY,_Fields.REWRITE_ENABLED,_Fields.CREATION_METADATA,_Fields.CAT_NAME,_Fields.OWNER_TYPE,_Fields.WRITE_ID,_Fields.IS_STATS_COMPLIANT,_Fields.COL_STATS}; + private static final int __VERSION_ISSET_ID = 8; + private short __isset_bitfield = 0; + private static final _Fields optionals[] = {_Fields.ID,_Fields.PRIVILEGES,_Fields.TEMPORARY,_Fields.REWRITE_ENABLED,_Fields.CREATION_METADATA,_Fields.CAT_NAME,_Fields.OWNER_TYPE,_Fields.WRITE_ID,_Fields.IS_STATS_COMPLIANT,_Fields.COL_STATS,_Fields.VERSION}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -276,6 +282,8 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); tmpMap.put(_Fields.COL_STATS, new org.apache.thrift.meta_data.FieldMetaData("colStats", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT , "ColumnStatistics"))); + tmpMap.put(_Fields.VERSION, new org.apache.thrift.meta_data.FieldMetaData("version", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Table.class, metaDataMap); } @@ -287,6 +295,8 @@ public Table() { this.writeId = -1L; + this.version = 0L; + } public Table( @@ -381,6 +391,7 @@ public Table(Table other) { if (other.isSetColStats()) { this.colStats = other.colStats; } + this.version = other.version; } public Table deepCopy() { @@ -420,6 +431,8 @@ public void clear() { setIsStatsCompliantIsSet(false); this.isStatsCompliant = false; this.colStats = null; + this.version = 0L; + } public long getId() { @@ -954,6 +967,28 @@ public void setColStatsIsSet(boolean value) { } } + public long getVersion() { + return this.version; + } + + public void setVersion(long version) { + this.version = version; + setVersionIsSet(true); + } + + public void unsetVersion() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __VERSION_ISSET_ID); + } + + /** Returns true if field version is set (has been assigned a value) and false otherwise */ + public boolean isSetVersion() { + return EncodingUtils.testBit(__isset_bitfield, __VERSION_ISSET_ID); + } + + public void setVersionIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __VERSION_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case ID: @@ -1132,6 +1167,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case VERSION: + if (value == null) { + unsetVersion(); + } else { + setVersion((Long)value); + } + break; + } } @@ -1203,6 +1246,9 @@ public Object getFieldValue(_Fields field) { case COL_STATS: return getColStats(); + case VERSION: + return getVersion(); + } throw new IllegalStateException(); } @@ -1258,6 +1304,8 @@ public boolean isSet(_Fields field) { return isSetIsStatsCompliant(); case COL_STATS: return isSetColStats(); + case VERSION: + return isSetVersion(); } throw new IllegalStateException(); } @@ -1473,6 +1521,15 @@ public boolean equals(Table that) { return false; } + boolean this_present_version = true && this.isSetVersion(); + boolean that_present_version = true && that.isSetVersion(); + if (this_present_version || that_present_version) { + if (!(this_present_version && that_present_version)) + return false; + if (this.version != that.version) + return false; + } + return true; } @@ -1590,6 +1647,11 @@ public int hashCode() { if (present_colStats) list.add(colStats); + boolean present_version = true && (isSetVersion()); + list.add(present_version); + if (present_version) + list.add(version); + return list.hashCode(); } @@ -1821,6 +1883,16 @@ public int compareTo(Table other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetVersion()).compareTo(other.isSetVersion()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetVersion()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.version, other.version); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -2004,6 +2076,12 @@ public String toString() { } first = false; } + if (isSetVersion()) { + if (!first) sb.append(", "); + sb.append("version:"); + sb.append(this.version); + first = false; + } sb.append(")"); return sb.toString(); } @@ -2258,6 +2336,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Table struct) throw org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 23: // VERSION + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.version = iprot.readI64(); + struct.setVersionIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -2400,6 +2486,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, Table struct) thro oprot.writeFieldEnd(); } } + if (struct.isSetVersion()) { + oprot.writeFieldBegin(VERSION_FIELD_DESC); + oprot.writeI64(struct.version); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -2484,7 +2575,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Table struct) throw if (struct.isSetColStats()) { optionals.set(21); } - oprot.writeBitSet(optionals, 22); + if (struct.isSetVersion()) { + optionals.set(22); + } + oprot.writeBitSet(optionals, 23); if (struct.isSetId()) { oprot.writeI64(struct.id); } @@ -2564,12 +2658,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Table struct) throw if (struct.isSetColStats()) { struct.colStats.write(oprot); } + if (struct.isSetVersion()) { + oprot.writeI64(struct.version); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, Table struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(22); + BitSet incoming = iprot.readBitSet(23); if (incoming.get(0)) { struct.id = iprot.readI64(); struct.setIdIsSet(true); @@ -2683,6 +2780,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Table struct) throws struct.colStats.read(iprot); struct.setColStatsIsSet(true); } + if (incoming.get(22)) { + struct.version = iprot.readI64(); + struct.setVersionIsSet(true); + } } } diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php index 1ae447d206ee72dda4f4c583394be93220756990..0fcdbe753964c02744657e9d066a7ead3f5c25e5 100644 --- standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php +++ standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php @@ -6846,6 +6846,10 @@ class Table { * @var \metastore\ColumnStatistics */ public $colStats = null; + /** + * @var int + */ + public $version = 0; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -6955,6 +6959,10 @@ class Table { 'type' => TType::STRUCT, 'class' => '\metastore\ColumnStatistics', ), + 23 => array( + 'var' => 'version', + 'type' => TType::I64, + ), ); } if (is_array($vals)) { @@ -7024,6 +7032,9 @@ class Table { if (isset($vals['colStats'])) { $this->colStats = $vals['colStats']; } + if (isset($vals['version'])) { + $this->version = $vals['version']; + } } } @@ -7228,6 +7239,13 @@ class Table { $xfer += $input->skip($ftype); } break; + case 23: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->version); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -7388,6 +7406,11 @@ class Table { $xfer += $this->colStats->write($output); $xfer += $output->writeFieldEnd(); } + if ($this->version !== null) { + $xfer += $output->writeFieldBegin('version', TType::I64, 23); + $xfer += $output->writeI64($this->version); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; @@ -7442,6 +7465,10 @@ class Partition { * @var bool */ public $isStatsCompliant = null; + /** + * @var int + */ + public $version = 0; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -7504,6 +7531,10 @@ class Partition { 'var' => 'isStatsCompliant', 'type' => TType::BOOL, ), + 12 => array( + 'var' => 'version', + 'type' => TType::I64, + ), ); } if (is_array($vals)) { @@ -7540,6 +7571,9 @@ class Partition { if (isset($vals['isStatsCompliant'])) { $this->isStatsCompliant = $vals['isStatsCompliant']; } + if (isset($vals['version'])) { + $this->version = $vals['version']; + } } } @@ -7664,6 +7698,13 @@ class Partition { $xfer += $input->skip($ftype); } break; + case 12: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->version); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -7763,6 +7804,11 @@ class Partition { $xfer += $output->writeBool($this->isStatsCompliant); $xfer += $output->writeFieldEnd(); } + if ($this->version !== null) { + $xfer += $output->writeFieldBegin('version', TType::I64, 12); + $xfer += $output->writeI64($this->version); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 06938b4071a7e439afb7b0804b81643c23a4fb52..c178e57c251bf9631f7a0f6ca8ce10dd899a3fef 100644 --- standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -4792,6 +4792,7 @@ class Table: - writeId - isStatsCompliant - colStats + - version """ thrift_spec = ( @@ -4818,9 +4819,10 @@ class Table: (20, TType.I64, 'writeId', None, -1, ), # 20 (21, TType.BOOL, 'isStatsCompliant', None, None, ), # 21 (22, TType.STRUCT, 'colStats', (ColumnStatistics, ColumnStatistics.thrift_spec), None, ), # 22 + (23, TType.I64, 'version', None, 0, ), # 23 ) - def __init__(self, id=None, tableName=None, dbName=None, owner=None, createTime=None, lastAccessTime=None, retention=None, sd=None, partitionKeys=None, parameters=None, viewOriginalText=None, viewExpandedText=None, tableType=None, privileges=None, temporary=thrift_spec[15][4], rewriteEnabled=None, creationMetadata=None, catName=None, ownerType=thrift_spec[19][4], writeId=thrift_spec[20][4], isStatsCompliant=None, colStats=None,): + def __init__(self, id=None, tableName=None, dbName=None, owner=None, createTime=None, lastAccessTime=None, retention=None, sd=None, partitionKeys=None, parameters=None, viewOriginalText=None, viewExpandedText=None, tableType=None, privileges=None, temporary=thrift_spec[15][4], rewriteEnabled=None, creationMetadata=None, catName=None, ownerType=thrift_spec[19][4], writeId=thrift_spec[20][4], isStatsCompliant=None, colStats=None, version=thrift_spec[23][4],): self.id = id self.tableName = tableName self.dbName = dbName @@ -4843,6 +4845,7 @@ def __init__(self, id=None, tableName=None, dbName=None, owner=None, createTime= self.writeId = writeId self.isStatsCompliant = isStatsCompliant self.colStats = colStats + self.version = version def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -4979,6 +4982,11 @@ def read(self, iprot): self.colStats.read(iprot) else: iprot.skip(ftype) + elif fid == 23: + if ftype == TType.I64: + self.version = iprot.readI64() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5084,6 +5092,10 @@ def write(self, oprot): oprot.writeFieldBegin('colStats', TType.STRUCT, 22) self.colStats.write(oprot) oprot.writeFieldEnd() + if self.version is not None: + oprot.writeFieldBegin('version', TType.I64, 23) + oprot.writeI64(self.version) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -5115,6 +5127,7 @@ def __hash__(self): value = (value * 31) ^ hash(self.writeId) value = (value * 31) ^ hash(self.isStatsCompliant) value = (value * 31) ^ hash(self.colStats) + value = (value * 31) ^ hash(self.version) return value def __repr__(self): @@ -5142,6 +5155,7 @@ class Partition: - catName - writeId - isStatsCompliant + - version """ thrift_spec = ( @@ -5157,9 +5171,10 @@ class Partition: (9, TType.STRING, 'catName', None, None, ), # 9 (10, TType.I64, 'writeId', None, -1, ), # 10 (11, TType.BOOL, 'isStatsCompliant', None, None, ), # 11 + (12, TType.I64, 'version', None, 0, ), # 12 ) - def __init__(self, values=None, dbName=None, tableName=None, createTime=None, lastAccessTime=None, sd=None, parameters=None, privileges=None, catName=None, writeId=thrift_spec[10][4], isStatsCompliant=None,): + def __init__(self, values=None, dbName=None, tableName=None, createTime=None, lastAccessTime=None, sd=None, parameters=None, privileges=None, catName=None, writeId=thrift_spec[10][4], isStatsCompliant=None, version=thrift_spec[12][4],): self.values = values self.dbName = dbName self.tableName = tableName @@ -5171,6 +5186,7 @@ def __init__(self, values=None, dbName=None, tableName=None, createTime=None, la self.catName = catName self.writeId = writeId self.isStatsCompliant = isStatsCompliant + self.version = version def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -5249,6 +5265,11 @@ def read(self, iprot): self.isStatsCompliant = iprot.readBool() else: iprot.skip(ftype) + elif fid == 12: + if ftype == TType.I64: + self.version = iprot.readI64() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5310,6 +5331,10 @@ def write(self, oprot): oprot.writeFieldBegin('isStatsCompliant', TType.BOOL, 11) oprot.writeBool(self.isStatsCompliant) oprot.writeFieldEnd() + if self.version is not None: + oprot.writeFieldBegin('version', TType.I64, 12) + oprot.writeI64(self.version) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -5330,6 +5355,7 @@ def __hash__(self): value = (value * 31) ^ hash(self.catName) value = (value * 31) ^ hash(self.writeId) value = (value * 31) ^ hash(self.isStatsCompliant) + value = (value * 31) ^ hash(self.version) return value def __repr__(self): diff --git standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb index e885194108bc7af3df14128507a7de0a36e53d1c..9e56d5a19fd84901efc4f05028ac3a8c2c48d96b 100644 --- standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -1128,6 +1128,7 @@ class Table WRITEID = 20 ISSTATSCOMPLIANT = 21 COLSTATS = 22 + VERSION = 23 FIELDS = { ID => {:type => ::Thrift::Types::I64, :name => 'id', :optional => true}, @@ -1151,7 +1152,8 @@ class Table OWNERTYPE => {:type => ::Thrift::Types::I32, :name => 'ownerType', :default => 1, :optional => true, :enum_class => ::PrincipalType}, WRITEID => {:type => ::Thrift::Types::I64, :name => 'writeId', :default => -1, :optional => true}, ISSTATSCOMPLIANT => {:type => ::Thrift::Types::BOOL, :name => 'isStatsCompliant', :optional => true}, - COLSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'colStats', :class => ::ColumnStatistics, :optional => true} + COLSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'colStats', :class => ::ColumnStatistics, :optional => true}, + VERSION => {:type => ::Thrift::Types::I64, :name => 'version', :default => 0, :optional => true} } def struct_fields; FIELDS; end @@ -1178,6 +1180,7 @@ class Partition CATNAME = 9 WRITEID = 10 ISSTATSCOMPLIANT = 11 + VERSION = 12 FIELDS = { VALUES => {:type => ::Thrift::Types::LIST, :name => 'values', :element => {:type => ::Thrift::Types::STRING}}, @@ -1190,7 +1193,8 @@ class Partition PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true}, CATNAME => {:type => ::Thrift::Types::STRING, :name => 'catName', :optional => true}, WRITEID => {:type => ::Thrift::Types::I64, :name => 'writeId', :default => -1, :optional => true}, - ISSTATSCOMPLIANT => {:type => ::Thrift::Types::BOOL, :name => 'isStatsCompliant', :optional => true} + ISSTATSCOMPLIANT => {:type => ::Thrift::Types::BOOL, :name => 'isStatsCompliant', :optional => true}, + VERSION => {:type => ::Thrift::Types::I64, :name => 'version', :default => 0, :optional => true} } def struct_fields; FIELDS; end diff --git standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index c58015dae4a8af26e275d470da263340e8d2c779..333567a1e29d73844beaad47bc98c9b0e0ccee69 100644 --- standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -448,7 +448,8 @@ struct Table { 19: optional PrincipalType ownerType = PrincipalType.USER, // owner type of this table (default to USER for backward compatibility) 20: optional i64 writeId=-1, 21: optional bool isStatsCompliant, - 22: optional ColumnStatistics colStats // column statistics for table + 22: optional ColumnStatistics colStats, // column statistics for table + 23: optional i64 version=0 // Version number which is incremented for every alter operation } struct Partition { @@ -462,7 +463,8 @@ struct Partition { 8: optional PrincipalPrivilegeSet privileges, 9: optional string catName, 10: optional i64 writeId=-1, - 11: optional bool isStatsCompliant + 11: optional bool isStatsCompliant, + 12: optional i64 version=0 // Version number which is incremented for every alter operation } struct PartitionWithoutSD { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java index f4e0c41b7c0bf31ea11130a525f3c20a8c61b96b..d04079c245d3d75a51bfd7c4263ae5a9d9e494ec 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java @@ -19,6 +19,7 @@ import java.util.List; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; @@ -115,7 +116,7 @@ void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, * @throws MetaException */ @Deprecated - Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, + Pair alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List part_vals, final Partition new_part, EnvironmentContext environmentContext) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; @@ -143,7 +144,7 @@ Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, * @throws AlreadyExistsException * @throws MetaException */ - Partition alterPartition(final RawStore msdb, Warehouse wh, final String catName, + Pair alterPartition(final RawStore msdb, Warehouse wh, final String catName, final String dbname, final String name, final List part_vals, final Partition new_part, EnvironmentContext environmentContext, IHMSHandler handler, String validWriteIds) @@ -196,7 +197,7 @@ Partition alterPartition(final RawStore msdb, Warehouse wh, final String catName * @throws AlreadyExistsException * @throws MetaException */ - List alterPartitions(final RawStore msdb, Warehouse wh, final String catName, + Pair, List> alterPartitions(final RawStore msdb, Warehouse wh, final String catName, final String dbname, final String name, final List new_parts, EnvironmentContext environmentContext, String writeIdList, long writeId, IHMSHandler handler) diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 617c7bc012a748f118683774003b0e5560b49e75..354efd0a8e54ad4265599cdec665aaa10a6b22bd 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -21,6 +21,8 @@ import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; @@ -127,7 +129,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam boolean isPartitionedTable = false; Table oldt = null; - + Table alteredTbl = null; List transactionalListeners = handler.getTransactionalListeners(); List listeners = handler.getListeners(); Map txnAlterTableEventResponses = Collections.emptyMap(); @@ -280,7 +282,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam } } // Do not verify stats parameters on a partitioned table. - msdb.alterTable(catName, dbname, name, newt, null); + alteredTbl = msdb.alterTable(catName, dbname, name, newt, null); // alterPartition is only for changing the partition location in the table rename if (dataWasMoved) { @@ -311,8 +313,8 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam writeIdList, newt.getWriteId()); } } else { - alterTableUpdateTableColumnStats( - msdb, oldt, newt, environmentContext, writeIdList, conf, null); + alteredTbl = alterTableUpdateTableColumnStats( + msdb, oldt, newt, environmentContext, writeIdList, conf, null).getLeft(); } } else { // operations other than table rename @@ -345,22 +347,22 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam } } // Don't validate table-level stats for a partitoned table. - msdb.alterTable(catName, dbname, name, newt, null); + alteredTbl = msdb.alterTable(catName, dbname, name, newt, null); } else { LOG.warn("Alter table not cascaded to partitions."); - alterTableUpdateTableColumnStats( - msdb, oldt, newt, environmentContext, writeIdList, conf, null); + alteredTbl = alterTableUpdateTableColumnStats( + msdb, oldt, newt, environmentContext, writeIdList, conf, null).getLeft(); } } else { - alterTableUpdateTableColumnStats( - msdb, oldt, newt, environmentContext, writeIdList, conf, null); + alteredTbl = alterTableUpdateTableColumnStats( + msdb, oldt, newt, environmentContext, writeIdList, conf, null).getLeft(); } } if (transactionalListeners != null && !transactionalListeners.isEmpty()) { txnAlterTableEventResponses = MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_TABLE, - new AlterTableEvent(oldt, newt, false, true, + new AlterTableEvent(oldt, alteredTbl, false, true, newt.getWriteId(), handler), environmentContext); } @@ -406,7 +408,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam // make this call whether the event failed or succeeded. To make this behavior consistent, // this call is made for failed events also. MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.ALTER_TABLE, - new AlterTableEvent(oldt, newt, false, success, newt.getWriteId(), handler), + new AlterTableEvent(oldt, alteredTbl, false, success, newt.getWriteId(), handler), environmentContext, txnAlterTableEventResponses, msdb); } } @@ -429,7 +431,7 @@ String getSimpleMessage(Exception ex) { } @Override - public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, + public Pair alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List part_vals, final Partition new_part, EnvironmentContext environmentContext) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { @@ -438,12 +440,13 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String } @Override - public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, String dbname, + public Pair alterPartition(RawStore msdb, Warehouse wh, String catName, String dbname, String name, List part_vals, final Partition new_part, EnvironmentContext environmentContext, IHMSHandler handler, String validWriteIds) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { boolean success = false; Partition oldPart; + Partition alteredPartition = null; List transactionalListeners = null; if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); @@ -483,12 +486,12 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str updateOrGetPartitionColumnStats(msdb, catName, dbname, name, new_part.getValues(), oldPart.getSd().getCols(), tbl, new_part, null, null); } - msdb.alterPartition( + alteredPartition = msdb.alterPartition( catName, dbname, name, new_part.getValues(), new_part, validWriteIds); if (transactionalListeners != null && !transactionalListeners.isEmpty()) { MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_PARTITION, - new AlterPartitionEvent(oldPart, new_part, tbl, false, + new AlterPartitionEvent(oldPart, alteredPartition, tbl, false, true, new_part.getWriteId(), handler), environmentContext); @@ -506,7 +509,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str msdb.rollbackTransaction(); } } - return oldPart; + return new ImmutablePair<>(oldPart, alteredPartition); } //rename partition @@ -621,7 +624,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str String newPartName = Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()); ColumnStatistics cs = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, oldPart.getValues(), oldPart.getSd().getCols(), tbl, new_part, null, null); - msdb.alterPartition(catName, dbname, name, part_vals, new_part, validWriteIds); + alteredPartition = msdb.alterPartition(catName, dbname, name, part_vals, new_part, validWriteIds); if (cs != null) { cs.getStatsDesc().setPartName(newPartName); try { @@ -637,7 +640,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str if (transactionalListeners != null && !transactionalListeners.isEmpty()) { MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventMessage.EventType.ALTER_PARTITION, - new AlterPartitionEvent(oldPart, new_part, tbl, false, + new AlterPartitionEvent(oldPart, alteredPartition, tbl, false, true, new_part.getWriteId(), handler), environmentContext); } @@ -663,7 +666,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str } } } - return oldPart; + return new ImmutablePair<>(oldPart, alteredPartition); } @Deprecated @@ -673,11 +676,11 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str EnvironmentContext environmentContext) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { return alterPartitions(msdb, wh, DEFAULT_CATALOG_NAME, dbname, name, new_parts, - environmentContext, null, -1, null); + environmentContext, null, -1, null).getLeft(); } @Override - public List alterPartitions(final RawStore msdb, Warehouse wh, final String catName, + public Pair, List> alterPartitions(final RawStore msdb, Warehouse wh, final String catName, final String dbname, final String name, final List new_parts, EnvironmentContext environmentContext, @@ -687,6 +690,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str List oldParts = new ArrayList<>(); List> partValsList = new ArrayList<>(); List transactionalListeners = null; + List alteredPartitions = null; if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); } @@ -731,9 +735,10 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str } } - msdb.alterPartitions(catName, dbname, name, partValsList, new_parts, writeId, writeIdList); + alteredPartitions = + msdb.alterPartitions(catName, dbname, name, partValsList, new_parts, writeId, writeIdList); Iterator oldPartsIt = oldParts.iterator(); - for (Partition newPart : new_parts) { + for (Partition newPart : alteredPartitions) { Partition oldPart; if (oldPartsIt.hasNext()) { oldPart = oldPartsIt.next(); @@ -758,7 +763,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str } } - return oldParts; + return new ImmutablePair<>(oldParts, alteredPartitions); } private boolean checkPartialPartKeysEqual(List oldPartKeys, @@ -799,7 +804,7 @@ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { } @VisibleForTesting - public static List alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable, + public static Pair> alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable, EnvironmentContext ec, String validWriteIds, Configuration conf, List deletedCols) throws MetaException, InvalidObjectException { String catName = normalizeIdentifier(oldTable.isSetCatName() ? oldTable.getCatName() : @@ -811,7 +816,7 @@ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { List newStatsObjs = new ArrayList<>(); //if its not called from cahced store then update the table boolean doAlterTable = deletedCols == null; - + Table alteredTbl = null; try { List oldCols = oldTable.getSd().getCols(); List newCols = newTable.getSd().getCols(); @@ -874,7 +879,7 @@ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { if (doAlterTable) { // Change to new table and append stats for the new table - msdb.alterTable(catName, dbName, tableName, newTable, validWriteIds); + alteredTbl = msdb.alterTable(catName, dbName, tableName, newTable, validWriteIds); if (updateColumnStats && !newStatsObjs.isEmpty()) { ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); statsDesc.setDbName(newDbName); @@ -889,7 +894,7 @@ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { //should not happen since the input were verified before passed in throw new InvalidObjectException("Invalid inputs to update table column stats: " + e); } - return newStatsObjs; + return new ImmutablePair<>(alteredTbl, newStatsObjs); } public static ColumnStatistics updateOrGetPartitionColumnStats( diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index c0b1d87462412e8ef52b58c3d117350c9c157446..86632f297087e47f74a696e47400a0419e82d816 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -72,6 +72,7 @@ import com.google.common.collect.Lists; import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -5005,9 +5006,10 @@ private void rename_partition(String catName, String db_name, String tbl_name, partitionValidationPattern); } - oldPart = alterHandler.alterPartition(getMS(), wh, catName, db_name, tbl_name, + Pair alterPartitionResult = alterHandler.alterPartition(getMS(), wh, catName, db_name, tbl_name, part_vals, new_part, envContext, this, validWriteIds); - + oldPart = alterPartitionResult.getLeft(); + new_part = alterPartitionResult.getRight(); // Only fetch the table if we actually have a listener Table table = null; if (!listeners.isEmpty()) { @@ -5088,6 +5090,8 @@ private void alter_partitions_with_environment_context(String catName, String db // all partitions are altered atomically // all prehooks are fired together followed by all post hooks List oldParts = null; + List alteredParts = null; + Pair, List> alterPartitionResult = null; Exception ex = null; try { for (Partition tmpPart : new_parts) { @@ -5097,12 +5101,14 @@ private void alter_partitions_with_environment_context(String catName, String db } firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, null, tmpPart, this)); } - oldParts = alterHandler.alterPartitions(getMS(), wh, + alterPartitionResult = alterHandler.alterPartitions(getMS(), wh, catName, db_name, tbl_name, new_parts, environmentContext, writeIdList, writeId, this); + oldParts = alterPartitionResult.getLeft(); + alteredParts = alterPartitionResult.getRight(); Iterator olditr = oldParts.iterator(); // Only fetch the table if we have a listener that needs it. Table table = null; - for (Partition tmpPart : new_parts) { + for (Partition tmpPart : alteredParts) { Partition oldTmpPart; if (olditr.hasNext()) { oldTmpPart = olditr.next(); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 9f721243c94d48eef35acdcbd0c2e143ab6d23ec..85c2ccf2129f219b1da5c0ec1677ff46b5be8680 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -62,6 +62,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Strings; +import com.sun.tools.javac.util.Assert; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -1780,6 +1781,7 @@ private Table convertToTable(MTable mtbl) throws MetaException { t.setRewriteEnabled(mtbl.isRewriteEnabled()); t.setCatName(mtbl.getDatabase().getCatalogName()); t.setWriteId(mtbl.getWriteId()); + t.setVersion(mtbl.getVersion()); return t; } @@ -2472,6 +2474,7 @@ private Partition convertToPart(MPartition mpart) throws MetaException { convertMap(mpart.getParameters())); p.setCatName(catName); p.setWriteId(mpart.getWriteId()); + p.setVersion(mpart.getVersion()); return p; } @@ -4166,6 +4169,7 @@ public Table alterTable(String catName, String dbname, String name, Table newTab dbname = normalizeIdentifier(dbname); catName = normalizeIdentifier(catName); MTable newt = convertToMTable(newTable); + lockTableForVersionUpdate(newt.getId()); if (newt == null) { throw new InvalidObjectException("new table is invalid"); } @@ -4224,6 +4228,8 @@ public Table alterTable(String catName, String dbname, String name, Table newTab oldt.setWriteId(newTable.getWriteId()); } } + + oldt.incrementVersion(); newTable = convertToTable(oldt); // commit the changes @@ -4236,6 +4242,33 @@ public Table alterTable(String catName, String dbname, String name, Table newTab return newTable; } + void lockTableForVersionUpdate(long tblId) throws MetaException { + String selectQuery = + "select * from \"TBLS\" where \"TBL_ID\"="+tblId; + String selectForUpdateQuery = sqlGenerator.addForUpdateClause(selectQuery); + new RetryingExecutor(conf, () -> { + prepareQuotes(); + Query query = pm.newQuery("javax.jdo.query.SQL", selectForUpdateQuery); + query.setUnique(true); + // only need to execute it to get db Lock + query.execute(); + query.closeAll(); + }).run(); + } + + void lockPartitionForVersionUpdate(long tableId, String partitionName) throws MetaException { + String selectQuery = + "select * from \"PARTITIONS\" where \"TBL_ID\"="+tableId+" and \"PART_NAME\"='" +partitionName+ "'"; + String selectForUpdateQuery = sqlGenerator.addForUpdateClause(selectQuery); + new RetryingExecutor(conf, () -> { + prepareQuotes(); + Query query = pm.newQuery("javax.jdo.query.SQL", selectForUpdateQuery); + query.setUnique(true); + // only need to execute it to get db Lock + query.execute(); + query.closeAll(); + }).run(); + } /** * Verifies that the stats JSON string is unchanged for alter table (txn stats). * @return Error message with the details of the change, or null if the value has not changed. @@ -4316,6 +4349,7 @@ private Partition alterPartitionNoTxn(String catName, String dbname, String name if (oldp == null || newp == null) { throw new InvalidObjectException("partition does not exist."); } + lockPartitionForVersionUpdate(table.getId(), oldp.getPartitionName()); oldp.setValues(newp.getValues()); oldp.setPartitionName(newp.getPartitionName()); boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters()); @@ -4353,7 +4387,7 @@ private Partition alterPartitionNoTxn(String catName, String dbname, String name oldp.setWriteId(newPart.getWriteId()); } } - + oldp.incrementVersion(); oldCd.t = oldCD; return convertToPart(oldp); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 7ad4bd240b4790f74a095af9e98a511c8732a956..c7933a0ca7bcb70bc3c399706ff48e826d44f0fa 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -233,7 +233,7 @@ static private void updateStatsForTable(RawStore rawStore, Table before, Table a boolean needUpdateAggrStat = false; List statisticsObjs = HiveAlterHandler.alterTableUpdateTableColumnStats(rawStore, before, - after,null, null, rawStore.getConf(), deletedCols); + after,null, null, rawStore.getConf(), deletedCols).getRight(); if (colStats != null) { sharedCache.updateTableColStatsInCache(catalogName, dbName, tableName, statisticsObjs); needUpdateAggrStat = true; diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java index 267c9e8e5acd7f3b3666f9a68780091c5e792380..07b98a380710d0a3cf352fec9042201cad7ae160 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java @@ -31,7 +31,8 @@ private MStorageDescriptor sd; private Map parameters; private long writeId; - + private long version; + public MPartition() {} /** @@ -159,4 +160,12 @@ public long getWriteId() { public void setWriteId(long writeId) { this.writeId = writeId; } + + public long getVersion() { + return version; + } + + public void incrementVersion() { + ++this.version; + } } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java index 92182ae257c90dbae61de256f473c528f607e3ee..a333b6c8d8b54d135393944beb0a61914140fe7e 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java @@ -40,6 +40,7 @@ private boolean rewriteEnabled; private String tableType; private long writeId; + private long version; public MTable() {} @@ -288,4 +289,12 @@ public void setWriteId(long writeId) { public long getId() { return id; } + + public long getVersion() { + return version; + } + + public void incrementVersion() { + ++this.version; + } } diff --git standalone-metastore/metastore-server/src/main/resources/package.jdo standalone-metastore/metastore-server/src/main/resources/package.jdo index 117a2f825e5bf0f5138df2475f6c85022baf4b88..da74dd4e92839607f6b8303a0409f02ec62f419c 100644 --- standalone-metastore/metastore-server/src/main/resources/package.jdo +++ standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -153,6 +153,9 @@ + + + @@ -477,6 +480,9 @@ + + + diff --git standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql index 2f102a253ed8d475751eb7d0fbf250b85cc2c457..38fc1068870e028e0eb3b98c2a1ccab07dd16dfd 100644 --- standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -48,7 +48,7 @@ CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); -CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "WRITE_ID" BIGINT DEFAULT 0); +CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "WRITE_ID" BIGINT DEFAULT 0, "VERSION" BIGINT DEFAULT 0 ); CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128), "SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS" VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER); @@ -76,7 +76,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), " CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128)); -CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N', "WRITE_ID" BIGINT DEFAULT 0); +CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N', "WRITE_ID" BIGINT DEFAULT 0, "VERSION" BIGINT DEFAULT 0 ); CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); diff --git standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql index 8ad56fc03398705e8f1a611483b4ecbd7b601deb..04a4fe873b2f9922e4a9a909c286d7c575267c32 100644 --- standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql +++ standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql @@ -8,7 +8,8 @@ ALTER TABLE "APP"."WM_RESOURCEPLAN" ADD NS VARCHAR(128); UPDATE "APP"."WM_RESOURCEPLAN" SET NS = 'default' WHERE NS IS NULL; DROP INDEX "APP"."UNIQUE_WM_RESOURCEPLAN"; CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN" ("NS", "NAME"); - +ALTER TABLE "APP"."TBLS" ADD VERSION bigint DEFAULT 0; +ALTER TABLE "APP"."PARTITIONS" ADD VERSION bigint DEFAULT 0; -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 895bf016d390f389f0cb3323a69347d610404e28..75ead3bbe0f9f756d200bc504afdd1757ecc25ea 100644 --- standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -146,7 +146,8 @@ CREATE TABLE PARTITIONS PART_NAME nvarchar(767) NULL, SD_ID bigint NULL, TBL_ID bigint NULL, - WRITE_ID bigint NOT NULL DEFAULT 0); + WRITE_ID bigint NOT NULL DEFAULT 0, + VERSION bigint NOT NULL DEFAULT 0); ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); @@ -378,7 +379,8 @@ CREATE TABLE TBLS VIEW_EXPANDED_TEXT text NULL, VIEW_ORIGINAL_TEXT text NULL, IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0, - WRITE_ID bigint NOT NULL DEFAULT 0); + WRITE_ID bigint NOT NULL DEFAULT 0, + VERSION bigint NOT NULL DEFAULT 0); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); diff --git standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql index edde08db9ef7ee01800c7cc3a04c813014abdd18..ccc1477c15c38816758197feee972e05d7959307 100644 --- standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql +++ standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql @@ -9,6 +9,8 @@ ALTER TABLE WM_RESOURCEPLAN ADD NS nvarchar(128); UPDATE WM_RESOURCEPLAN SET NS = 'default' WHERE NS IS NULL; DROP INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN; CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NS", "NAME"); +ALTER TABLE TBLS ADD VERSION bigint NOT NULL DEFAULT 0; +ALTER TABLE PARTITIONS ADD VERSION bigint NOT NULL DEFAULT 0; -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 5466537c45662f191a726341b2a06f4fab824b0a..aa5c923a532a73e08e7d91afe805f62f8ff2dbba 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -227,6 +227,7 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` ( `SD_ID` bigint(20) DEFAULT NULL, `TBL_ID` bigint(20) DEFAULT NULL, `WRITE_ID` bigint(20) DEFAULT 0, + `VERSION` bigint(20) DEFAULT 0, PRIMARY KEY (`PART_ID`), UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`), KEY `PARTITIONS_N49` (`TBL_ID`), @@ -634,6 +635,7 @@ CREATE TABLE IF NOT EXISTS `TBLS` ( `VIEW_ORIGINAL_TEXT` mediumtext, `IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0, `WRITE_ID` bigint(20) DEFAULT 0, + `VERSION` bigint(20) DEFAULT 0, PRIMARY KEY (`TBL_ID`), UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), KEY `TBLS_N50` (`SD_ID`), diff --git standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql index 701acb00984c61f7511dcc48053890b154575d1f..915f72b0c8fde2e4b93ccc5c6970448835f9db01 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql @@ -10,6 +10,8 @@ ALTER TABLE `WM_RESOURCEPLAN` ADD `NS` varchar(128); UPDATE `WM_RESOURCEPLAN` SET `NS` = 'default' WHERE `NS` IS NULL; ALTER TABLE `WM_RESOURCEPLAN` DROP KEY `UNIQUE_WM_RESOURCEPLAN`; ALTER TABLE `WM_RESOURCEPLAN` ADD UNIQUE KEY `UNIQUE_WM_RESOURCEPLAN` (`NAME`, `NS`); +ALTER TABLE `TBLS` ADD VERSION bigint DEFAULT 0; +ALTER TABLE `PARTITIONS` ADD VERSION bigint DEFAULT 0; -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index 2a9c38f1863d76926dad634cf0812f5800076fdb..014064c2475574fe6256e16a63ae99618a6d6166 100644 --- standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -166,7 +166,8 @@ CREATE TABLE PARTITIONS PART_NAME VARCHAR2(767) NULL, SD_ID NUMBER NULL, TBL_ID NUMBER NULL, - WRITE_ID NUMBER DEFAULT 0 NOT NULL + WRITE_ID NUMBER DEFAULT 0 NOT NULL, + VERSION NUMBER DEFAULT 0 NOT NULL ); ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); @@ -402,7 +403,8 @@ CREATE TABLE TBLS VIEW_EXPANDED_TEXT CLOB NULL, VIEW_ORIGINAL_TEXT CLOB NULL, IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)), - WRITE_ID NUMBER DEFAULT 0 NOT NULL + WRITE_ID NUMBER DEFAULT 0 NOT NULL, + VERSION NUMBER DEFAULT 0 NOT NULL ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); diff --git standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql index b9f63313251ab1fa6278b862ed9e07e62b234c04..9908f9c5d42009cfcf50c158bfb031222ec87350 100644 --- standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql +++ standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql @@ -8,6 +8,8 @@ ALTER TABLE WM_RESOURCEPLAN ADD NS VARCHAR2(128); UPDATE WM_RESOURCEPLAN SET NS = 'default' WHERE NS IS NULL; DROP INDEX UNIQUE_WM_RESOURCEPLAN; CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN (NS, "NAME"); +ALTER TABLE TBLS ADD VERSION number DEFAULT 0 NOT NULL; +ALTER TABLE PARTITIONS ADD VERSION number DEFAULT 0 NOT NULL; -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index 0a359d9e2a576d819cee2a2dcd9a51b60a8dcb17..3a6c11226b5c8afa1ccdf15dfbd16294585837b4 100644 --- standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -171,7 +171,8 @@ CREATE TABLE "PARTITIONS" ( "PART_NAME" character varying(767) DEFAULT NULL::character varying, "SD_ID" bigint, "TBL_ID" bigint, - "WRITE_ID" bigint DEFAULT 0 + "WRITE_ID" bigint DEFAULT 0, + "VERSION" bigint DEFAULT 0 ); @@ -396,7 +397,8 @@ CREATE TABLE "TBLS" ( "VIEW_EXPANDED_TEXT" text, "VIEW_ORIGINAL_TEXT" text, "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false, - "WRITE_ID" bigint DEFAULT 0 + "WRITE_ID" bigint DEFAULT 0, + "VERSION" bigint DEFAULT 0 ); -- diff --git standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql index 0c36069d071d4b60cc338ba729da5d22e08ca8ca..d94cb5cf2958df7cfc55fd353e87e4aab3dd285f 100644 --- standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql +++ standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql @@ -10,6 +10,8 @@ ALTER TABLE "WM_RESOURCEPLAN" ADD "NS" character varying(128); UPDATE "WM_RESOURCEPLAN" SET "NS" = 'default' WHERE "NS" IS NULL; ALTER TABLE "WM_RESOURCEPLAN" DROP CONSTRAINT "UNIQUE_WM_RESOURCEPLAN"; ALTER TABLE ONLY "WM_RESOURCEPLAN" ADD CONSTRAINT "UNIQUE_WM_RESOURCEPLAN" UNIQUE ("NS", "NAME"); +ALTER TABLE "TBLS" ADD "VERSION" bigint DEFAULT 0; +ALTER TABLE "PARTITIONS" ADD "VERSION" bigint DEFAULT 0; -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1;