commit 4b63d958a4586d0e5f3b0b66fab98ae5bdf90ebb Author: Vihang Karajgaonkar Date: Thu Jan 3 10:56:05 2019 -0800 HIVE-21077 : Database should have creation time diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Database.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Database.java index 994797698a379e0b08604d73d2d6728a2fcee4df..f68fda98c773fd10f78799d7f40918494d80b83f 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Database.java +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Database.java @@ -46,6 +46,7 @@ private static final org.apache.thrift.protocol.TField OWNER_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("ownerName", org.apache.thrift.protocol.TType.STRING, (short)6); private static final org.apache.thrift.protocol.TField OWNER_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("ownerType", org.apache.thrift.protocol.TType.I32, (short)7); private static final org.apache.thrift.protocol.TField CATALOG_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catalogName", org.apache.thrift.protocol.TType.STRING, (short)8); + private static final org.apache.thrift.protocol.TField CREATE_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("createTime", org.apache.thrift.protocol.TType.I32, (short)9); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -61,6 +62,7 @@ private String ownerName; // optional private PrincipalType ownerType; // optional private String catalogName; // optional + private int createTime; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -75,7 +77,8 @@ * @see PrincipalType */ OWNER_TYPE((short)7, "ownerType"), - CATALOG_NAME((short)8, "catalogName"); + CATALOG_NAME((short)8, "catalogName"), + CREATE_TIME((short)9, "createTime"); private static final Map byName = new HashMap(); @@ -106,6 +109,8 @@ public static _Fields findByThriftId(int fieldId) { return OWNER_TYPE; case 8: // CATALOG_NAME return CATALOG_NAME; + case 9: // CREATE_TIME + return CREATE_TIME; default: return null; } @@ -146,7 +151,9 @@ public String getFieldName() { } // isset id assignments - private static final _Fields optionals[] = {_Fields.PRIVILEGES,_Fields.OWNER_NAME,_Fields.OWNER_TYPE,_Fields.CATALOG_NAME}; + private static final int __CREATETIME_ISSET_ID = 0; + private byte __isset_bitfield = 0; + private static final _Fields optionals[] = {_Fields.PRIVILEGES,_Fields.OWNER_NAME,_Fields.OWNER_TYPE,_Fields.CATALOG_NAME,_Fields.CREATE_TIME}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -168,6 +175,8 @@ public String getFieldName() { new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, PrincipalType.class))); tmpMap.put(_Fields.CATALOG_NAME, new org.apache.thrift.meta_data.FieldMetaData("catalogName", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.CREATE_TIME, new org.apache.thrift.meta_data.FieldMetaData("createTime", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Database.class, metaDataMap); } @@ -192,6 +201,7 @@ public Database( * Performs a deep copy on other. */ public Database(Database other) { + __isset_bitfield = other.__isset_bitfield; if (other.isSetName()) { this.name = other.name; } @@ -217,6 +227,7 @@ public Database(Database other) { if (other.isSetCatalogName()) { this.catalogName = other.catalogName; } + this.createTime = other.createTime; } public Database deepCopy() { @@ -233,6 +244,8 @@ public void clear() { this.ownerName = null; this.ownerType = null; this.catalogName = null; + setCreateTimeIsSet(false); + this.createTime = 0; } public String getName() { @@ -438,6 +451,28 @@ public void setCatalogNameIsSet(boolean value) { } } + public int getCreateTime() { + return this.createTime; + } + + public void setCreateTime(int createTime) { + this.createTime = createTime; + setCreateTimeIsSet(true); + } + + public void unsetCreateTime() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __CREATETIME_ISSET_ID); + } + + /** Returns true if field createTime is set (has been assigned a value) and false otherwise */ + public boolean isSetCreateTime() { + return EncodingUtils.testBit(__isset_bitfield, __CREATETIME_ISSET_ID); + } + + public void setCreateTimeIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __CREATETIME_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case NAME: @@ -504,6 +539,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case CREATE_TIME: + if (value == null) { + unsetCreateTime(); + } else { + setCreateTime((Integer)value); + } + break; + } } @@ -533,6 +576,9 @@ public Object getFieldValue(_Fields field) { case CATALOG_NAME: return getCatalogName(); + case CREATE_TIME: + return getCreateTime(); + } throw new IllegalStateException(); } @@ -560,6 +606,8 @@ public boolean isSet(_Fields field) { return isSetOwnerType(); case CATALOG_NAME: return isSetCatalogName(); + case CREATE_TIME: + return isSetCreateTime(); } throw new IllegalStateException(); } @@ -649,6 +697,15 @@ public boolean equals(Database that) { return false; } + boolean this_present_createTime = true && this.isSetCreateTime(); + boolean that_present_createTime = true && that.isSetCreateTime(); + if (this_present_createTime || that_present_createTime) { + if (!(this_present_createTime && that_present_createTime)) + return false; + if (this.createTime != that.createTime) + return false; + } + return true; } @@ -696,6 +753,11 @@ public int hashCode() { if (present_catalogName) list.add(catalogName); + boolean present_createTime = true && (isSetCreateTime()); + list.add(present_createTime); + if (present_createTime) + list.add(createTime); + return list.hashCode(); } @@ -787,6 +849,16 @@ public int compareTo(Database other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetCreateTime()).compareTo(other.isSetCreateTime()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCreateTime()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.createTime, other.createTime); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -878,6 +950,12 @@ public String toString() { } first = false; } + if (isSetCreateTime()) { + if (!first) sb.append(", "); + sb.append("createTime:"); + sb.append(this.createTime); + first = false; + } sb.append(")"); return sb.toString(); } @@ -900,6 +978,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); @@ -1001,6 +1081,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Database struct) th org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 9: // CREATE_TIME + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.createTime = iprot.readI32(); + struct.setCreateTimeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1070,6 +1158,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, Database struct) t oprot.writeFieldEnd(); } } + if (struct.isSetCreateTime()) { + oprot.writeFieldBegin(CREATE_TIME_FIELD_DESC); + oprot.writeI32(struct.createTime); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1112,7 +1205,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Database struct) th if (struct.isSetCatalogName()) { optionals.set(7); } - oprot.writeBitSet(optionals, 8); + if (struct.isSetCreateTime()) { + optionals.set(8); + } + oprot.writeBitSet(optionals, 9); if (struct.isSetName()) { oprot.writeString(struct.name); } @@ -1144,12 +1240,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Database struct) th if (struct.isSetCatalogName()) { oprot.writeString(struct.catalogName); } + if (struct.isSetCreateTime()) { + oprot.writeI32(struct.createTime); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, Database struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(8); + BitSet incoming = iprot.readBitSet(9); if (incoming.get(0)) { struct.name = iprot.readString(); struct.setNameIsSet(true); @@ -1194,6 +1293,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Database struct) thr struct.catalogName = iprot.readString(); struct.setCatalogNameIsSet(true); } + if (incoming.get(8)) { + struct.createTime = iprot.readI32(); + struct.setCreateTimeIsSet(true); + } } } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php index 2953a8f327eabdee42dbc66e0c65f89d17add59a..c2a6eaf4410033feecfa8e57ec840ec9a522d4f9 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php @@ -5404,6 +5404,10 @@ class Database { * @var string */ public $catalogName = null; + /** + * @var int + */ + public $createTime = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -5449,6 +5453,10 @@ class Database { 'var' => 'catalogName', 'type' => TType::STRING, ), + 9 => array( + 'var' => 'createTime', + 'type' => TType::I32, + ), ); } if (is_array($vals)) { @@ -5476,6 +5484,9 @@ class Database { if (isset($vals['catalogName'])) { $this->catalogName = $vals['catalogName']; } + if (isset($vals['createTime'])) { + $this->createTime = $vals['createTime']; + } } } @@ -5568,6 +5579,13 @@ class Database { $xfer += $input->skip($ftype); } break; + case 9: + if ($ftype == TType::I32) { + $xfer += $input->readI32($this->createTime); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -5637,6 +5655,11 @@ class Database { $xfer += $output->writeString($this->catalogName); $xfer += $output->writeFieldEnd(); } + if ($this->createTime !== null) { + $xfer += $output->writeFieldBegin('createTime', TType::I32, 9); + $xfer += $output->writeI32($this->createTime); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py index f8b862862de4dde8dce3d0dc5f70eafb67b02d2c..050a4754707d3289b2daf4a9cce1cc8a5a65d980 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -3955,6 +3955,7 @@ class Database: - ownerName - ownerType - catalogName + - createTime """ thrift_spec = ( @@ -3967,9 +3968,10 @@ class Database: (6, TType.STRING, 'ownerName', None, None, ), # 6 (7, TType.I32, 'ownerType', None, None, ), # 7 (8, TType.STRING, 'catalogName', None, None, ), # 8 + (9, TType.I32, 'createTime', None, None, ), # 9 ) - def __init__(self, name=None, description=None, locationUri=None, parameters=None, privileges=None, ownerName=None, ownerType=None, catalogName=None,): + def __init__(self, name=None, description=None, locationUri=None, parameters=None, privileges=None, ownerName=None, ownerType=None, catalogName=None, createTime=None,): self.name = name self.description = description self.locationUri = locationUri @@ -3978,6 +3980,7 @@ def __init__(self, name=None, description=None, locationUri=None, parameters=Non self.ownerName = ownerName self.ownerType = ownerType self.catalogName = catalogName + self.createTime = createTime def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -4035,6 +4038,11 @@ def read(self, iprot): self.catalogName = iprot.readString() else: iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -4081,6 +4089,10 @@ def write(self, oprot): oprot.writeFieldBegin('catalogName', TType.STRING, 8) oprot.writeString(self.catalogName) oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin('createTime', TType.I32, 9) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -4098,6 +4110,7 @@ def __hash__(self): value = (value * 31) ^ hash(self.ownerName) value = (value * 31) ^ hash(self.ownerType) value = (value * 31) ^ hash(self.catalogName) + value = (value * 31) ^ hash(self.createTime) return value def __repr__(self): diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb index dfc5d7b294c1df8d5c6b0e7d676a77ba1181e076..0f79e33aa97f6bc0e5003fb6f70e1dd57ecd3c4f 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -961,6 +961,7 @@ class Database OWNERNAME = 6 OWNERTYPE = 7 CATALOGNAME = 8 + CREATETIME = 9 FIELDS = { NAME => {:type => ::Thrift::Types::STRING, :name => 'name'}, @@ -970,7 +971,8 @@ class Database PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true}, OWNERNAME => {:type => ::Thrift::Types::STRING, :name => 'ownerName', :optional => true}, OWNERTYPE => {:type => ::Thrift::Types::I32, :name => 'ownerType', :optional => true, :enum_class => ::PrincipalType}, - CATALOGNAME => {:type => ::Thrift::Types::STRING, :name => 'catalogName', :optional => true} + CATALOGNAME => {:type => ::Thrift::Types::STRING, :name => 'catalogName', :optional => true}, + CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime', :optional => true} } def struct_fields; FIELDS; end diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 7d09a5c296a8ae924d61b200b4cb9135440fd9a0..cdbe435f1b8047a8a7f3e4c3646faf1df5dede38 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -379,7 +379,8 @@ struct Database { 5: optional PrincipalPrivilegeSet privileges, 6: optional string ownerName, 7: optional PrincipalType ownerType, - 8: optional string catalogName + 8: optional string catalogName, + 9: optional i32 createTime // creation time of the database } // This object holds the information needed by SerDes diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index a9398ae1e79404a15894aa42f451df5d18ed3e4c..57de764f066fea2404c18403839259741fe114e3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -778,10 +778,11 @@ private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObje ms.getDatabase(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME); } catch (NoSuchObjectException e) { Database db = new Database(DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT, - wh.getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null); + wh.getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null); db.setOwnerName(PUBLIC); db.setOwnerType(PrincipalType.ROLE); db.setCatalogName(DEFAULT_CATALOG_NAME); + db.setCreateTime((int) System.currentTimeMillis() / 1000); ms.createDatabase(db); } } @@ -1062,8 +1063,9 @@ public void create_catalog(CreateCatalogRequest rqst) ms.createCatalog(catalog); // Create a default database inside the catalog - Database db = new Database(DEFAULT_DATABASE_NAME, "Default database for catalog " + - catalog.getName(), catalog.getLocationUri(), Collections.emptyMap()); + Database db = new Database(DEFAULT_DATABASE_NAME, + "Default database for catalog " + catalog.getName(), catalog.getLocationUri(), + Collections.emptyMap()); db.setCatalogName(catalog.getName()); create_database_core(ms, db); @@ -1286,7 +1288,7 @@ private void create_database_core(RawStore ms, final Database db) } Path dbPath = wh.determineDatabasePath(cat, db); db.setLocationUri(dbPath.toString()); - + db.setCreateTime((int) System.currentTimeMillis()/1000); boolean success = false; boolean madeDir = false; Map transactionalListenersResponses = Collections.emptyMap(); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 58dc6eefcb840d4dd70af7a47811fab1b5e696d9..b0cbe0168c77ca5aec96135f50a4a1261c682e62 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -371,7 +371,7 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ String queryTextDbSelector= "select " + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", " - + "\"OWNER_NAME\", \"OWNER_TYPE\", \"CTLG_NAME\" " + + "\"OWNER_NAME\", \"OWNER_TYPE\", \"CTLG_NAME\" , \"CREATE_TIME\"" + "FROM "+ DBS + " where \"NAME\" = ? and \"CTLG_NAME\" = ? "; Object[] params = new Object[] { dbName, catName }; @@ -425,6 +425,7 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ db.setOwnerType( (null == type || type.trim().isEmpty()) ? null : PrincipalType.valueOf(type)); db.setCatalogName(MetastoreDirectSqlUtils.extractSqlString(dbline[6])); + db.setCreateTime(MetastoreDirectSqlUtils.extractSqlInt(dbline[7])); db.setParameters(MetaStoreServerUtils.trimMapNulls(dbParams,convertMapNullsToEmptyStrings)); if (LOG.isDebugEnabled()){ LOG.debug("getDatabase: directsql returning db " + db.getName() diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index d43c0c1e70cffbebd39b05f89ec396227c58ac77..9a749b3dd39dbe8e3568f7c4cdd1e85b544c4237 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -674,6 +674,7 @@ public void createDatabase(Database db) throws InvalidObjectException, MetaExcep mdb.setOwnerName(db.getOwnerName()); PrincipalType ownerType = db.getOwnerType(); mdb.setOwnerType((null == ownerType ? PrincipalType.USER.name() : ownerType.name())); + mdb.setCreateTime(db.getCreateTime()); try { openTransaction(); pm.makePersistent(mdb); @@ -767,6 +768,7 @@ public Database getJDODatabase(String catName, String name) throws NoSuchObjectE PrincipalType principalType = (type == null) ? null : PrincipalType.valueOf(type); db.setOwnerType(principalType); db.setCatalogName(catName); + db.setCreateTime(mdb.getCreateTime()); return db; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java index f3d2182a04ab81417a55554ba58d9340721513e8..dcd57c4177b81c1e199be0f1ba1a7dc1d74fa0b2 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java @@ -41,6 +41,7 @@ private Map params = new HashMap<>(); private String ownerName; private PrincipalType ownerType; + private int createTime; public DatabaseBuilder() { } @@ -90,11 +91,17 @@ public DatabaseBuilder setOwnerType(PrincipalType ownerType) { return this; } + public DatabaseBuilder setCreateTime(int createTime) { + this.createTime = createTime; + return this; + } + public Database build(Configuration conf) throws MetaException { if (name == null) throw new MetaException("You must name the database"); if (catalogName == null) catalogName = MetaStoreUtils.getDefaultCatalog(conf); Database db = new Database(name, description, location, params); db.setCatalogName(catalogName); + db.setCreateTime(createTime); try { if (ownerName == null) ownerName = SecurityUtils.getUser(); db.setOwnerName(ownerName); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MDatabase.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MDatabase.java index 815b39c483b2233660310983d58194fb1ab2d107..8901eedb83ce9542cd0971727c5cee0973cf14f7 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MDatabase.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MDatabase.java @@ -40,6 +40,7 @@ private String ownerName; private String ownerType; private String catalogName; + private int createTime; /** * Default construction to keep jpox/jdo happy @@ -154,4 +155,12 @@ public String getCatalogName() { public void setCatalogName(String catalogName) { this.catalogName = catalogName; } + + public int getCreateTime() { + return createTime; + } + + public void setCreateTime(int createTime) { + this.createTime = createTime; + } } diff --git a/standalone-metastore/metastore-server/src/main/resources/package.jdo b/standalone-metastore/metastore-server/src/main/resources/package.jdo index caaec457194332a99d5cd57bef746e969dd38161..1a6378369f920d68f02f361192b7fa85eb58e3be 100644 --- a/standalone-metastore/metastore-server/src/main/resources/package.jdo +++ b/standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -65,6 +65,9 @@ + + + diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql index a3c4196dbff7e53be5317631b314983d16a99020..c4ce93b58cfea95b1b486b91549f2de2579ac163 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -22,7 +22,8 @@ CREATE TABLE "APP"."DBS" ( "NAME" VARCHAR(128), "OWNER_NAME" VARCHAR(128), "OWNER_TYPE" VARCHAR(10), - "CTLG_NAME" VARCHAR(256) NOT NULL + "CTLG_NAME" VARCHAR(256) NOT NULL, + "CREATE_TIME" INTEGER NOT NULL ); CREATE TABLE "APP"."TBL_PRIVS" ("TBL_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_PRIV" VARCHAR(128), "TBL_ID" BIGINT, "AUTHORIZER" VARCHAR(128)); diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql index bcaebd18accf86846ae44a6498046514575fc069..910c45a852883c3be2b0b08e75103fbbc9890330 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql @@ -9,7 +9,8 @@ UPDATE "APP"."WM_RESOURCEPLAN" SET NS = 'default' WHERE NS IS NULL; DROP INDEX "APP"."UNIQUE_WM_RESOURCEPLAN"; CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN" ("NS", "NAME"); - +-- HIVE-21077 +ALTER TABLE "APP"."DBS" ADD COLUMN CREATE_TIME bigint DEFAULT 0; -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 5ea1b4450d8258e841bb4af7381ca6fb0ba1a827..a133c7fc88bc49237cb65e9006037ce7675f6748 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -285,7 +285,8 @@ CREATE TABLE DBS "NAME" nvarchar(128) NULL, OWNER_NAME nvarchar(128) NULL, OWNER_TYPE nvarchar(10) NULL, - CTLG_NAME nvarchar(256) + CTLG_NAME nvarchar(256), + CREATE_TIME int NOT NULL ); ALTER TABLE DBS ADD CONSTRAINT DBS_PK PRIMARY KEY (DB_ID); diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql index edde08db9ef7ee01800c7cc3a04c813014abdd18..ec25b668a03cb66a9700cccd6710d333af78306d 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql @@ -10,6 +10,8 @@ UPDATE WM_RESOURCEPLAN SET NS = 'default' WHERE NS IS NULL; DROP INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN; CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NS", "NAME"); +-- HIVE-21077 +ALTER TABLE DBS ADD CREATE_TIME int NOT NULL DEFAULT 0; -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS MESSAGE; diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 5555a59c7d7e933d25d8d5af611e5b6aa0c0c19b..4f8f5ae8866a7b3e4f4559c9632fc6f9bd3310ac 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -100,6 +100,7 @@ CREATE TABLE IF NOT EXISTS `DBS` ( `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `OWNER_TYPE` varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `CTLG_NAME` varchar(256) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, PRIMARY KEY (`DB_ID`), UNIQUE KEY `UNIQUE_DATABASE` (`NAME`, `CTLG_NAME`), CONSTRAINT `CTLG_FK1` FOREIGN KEY (`CTLG_NAME`) REFERENCES `CTLGS` (`NAME`) diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql index 701acb00984c61f7511dcc48053890b154575d1f..619c579d36f7779f4523c97f258b028367c83132 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql @@ -11,6 +11,8 @@ UPDATE `WM_RESOURCEPLAN` SET `NS` = 'default' WHERE `NS` IS NULL; ALTER TABLE `WM_RESOURCEPLAN` DROP KEY `UNIQUE_WM_RESOURCEPLAN`; ALTER TABLE `WM_RESOURCEPLAN` ADD UNIQUE KEY `UNIQUE_WM_RESOURCEPLAN` (`NAME`, `NS`); +-- HIVE-21077 +ALTER TABLE `DBS` ADD `CREATE_TIME` int(11) DEFAULT 0 NOT NULL; -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS ' '; diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index b1980c5b83f16614845063516495188ebdd8c2a3..8b42415ba4fc66a2a3db028ef73449cbf0beedbb 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -92,7 +92,8 @@ CREATE TABLE DBS "NAME" VARCHAR2(128) NULL, OWNER_NAME VARCHAR2(128) NULL, OWNER_TYPE VARCHAR2(10) NULL, - CTLG_NAME VARCHAR2(256) + CTLG_NAME VARCHAR2(256), + CREATE_TIME NUMBER (10) NOT NULL ); ALTER TABLE DBS ADD CONSTRAINT DBS_PK PRIMARY KEY (DB_ID); diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql index b9f63313251ab1fa6278b862ed9e07e62b234c04..01366adb8bd868572ab175cd811fd72364a06bc9 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql @@ -9,6 +9,8 @@ UPDATE WM_RESOURCEPLAN SET NS = 'default' WHERE NS IS NULL; DROP INDEX UNIQUE_WM_RESOURCEPLAN; CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN (NS, "NAME"); +-- HIVE-21077 +ALTER TABLE DBS add CREATE_TIME NUMBER DEFAULT 0 NOT NULL; -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS Status from dual; diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index 9040005aa82b7a8cc5c01f257ecd47a7cc97e9b2..1d2af111c69ef78060b590f4a4b308e3aa5d9bed 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -77,7 +77,8 @@ CREATE TABLE "DBS" ( "NAME" character varying(128) DEFAULT NULL::character varying, "OWNER_NAME" character varying(128) DEFAULT NULL::character varying, "OWNER_TYPE" character varying(10) DEFAULT NULL::character varying, - "CTLG_NAME" varchar(256) + "CTLG_NAME" varchar(256), + "CREATE_TIME" bigint NOT NULL ); diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql index 0c36069d071d4b60cc338ba729da5d22e08ca8ca..530fd41bd13cb5ded8b8cba4eedb52702e2ffb52 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql @@ -11,6 +11,8 @@ UPDATE "WM_RESOURCEPLAN" SET "NS" = 'default' WHERE "NS" IS NULL; ALTER TABLE "WM_RESOURCEPLAN" DROP CONSTRAINT "UNIQUE_WM_RESOURCEPLAN"; ALTER TABLE ONLY "WM_RESOURCEPLAN" ADD CONSTRAINT "UNIQUE_WM_RESOURCEPLAN" UNIQUE ("NS", "NAME"); +-- HIVE-21077 +ALTER TABLE "DBS" ADD "CREATE_TIME" BIGINT DEFAULT 0; -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0'; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java index d323ac6c90ed20f092b4e179fdb1bed8602ecf63..791120477167b7253abe1c2e1d6a2d87195aa329 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDatabases.java @@ -126,6 +126,7 @@ public void testCreateGetDeleteDatabase() throws Exception { Database createdDatabase = client.getDatabase(database.getName()); // The createTime will be set on the server side, so the comparison should skip it + database.setCreateTime(createdDatabase.getCreateTime()); Assert.assertEquals("Comparing databases", database, createdDatabase); Assert.assertTrue("The directory should be created", metaStore.isPathExists( new Path(database.getLocationUri()))); @@ -210,6 +211,29 @@ public void testDefaultDatabaseData() throws Exception { Assert.assertEquals("Default database owner", "public", database.getOwnerName()); Assert.assertEquals("Default database owner type", PrincipalType.ROLE, database.getOwnerType()); Assert.assertNull("Default database privileges", database.getPrivileges()); + Assert.assertTrue("database create time should be set", database.isSetCreateTime()); + Assert.assertTrue("Database create time should be non-zero", database.getCreateTime() > 0); + } + + @Test + public void testDatabaseCreateTime() throws Exception { + // create db without specifying createtime + Database testDb = + new DatabaseBuilder().setName("test_create_time").create(client, metaStore.getConf()); + Database database = client.getDatabase("test_create_time"); + Assert.assertTrue("Database create time should have been set", + database.getCreateTime() > 0); + } + + @Test + public void testDatbaseCreateTime2() throws Exception { + // create db by providing a create time. Should be overridden, create time should + // always be set by metastore + Database testDb = + new DatabaseBuilder().setName("test_create_time").create(client, metaStore.getConf()); + Database database = client.getDatabase("test_create_time"); + Assert.assertTrue("Database create time should have been set", + database.getCreateTime() > 0); } @Test @@ -433,6 +457,7 @@ public void testAlterDatabase() throws Exception { .setDescription("dummy description 2") .addParam("param_key_1", "param_value_1_2") .addParam("param_key_2_3", "param_value_2_3") + .setCreateTime(originalDatabase.getCreateTime()) .build(metaStore.getConf()); client.alterDatabase(originalDatabase.getName(), newDatabase);