commit 91a70a530ef8c68ec56590990ac549beab7d3e15 Author: Bharath Krishna Date: Wed Dec 19 16:17:33 2018 -0800 HIVE-21027 : Add a configuration to include entire thrift objects in HMS notifications diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index fb0b2fe6fb9fd4b4c92a6a39f06f39a4641aaabd..ca062f47e3948dfa9e905fd2867da50c0e25ce49 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -547,6 +547,9 @@ public static ConfVars getMetaConf(String name) { + " The regexes are matched against each key of parameters map in Table or Partition object" + "present in HMS Notification. Any key-value pair whose key is matched with any regex will" +" be removed from Parameters map during Serialization of Table/Partition object."), + EVENT_NOTIFICATIONS_ADD_THRIFT_OBJECTS("metastore.notifications.add.thrift.objects", + "hive.metastore.notifications.add.thrift.objects", true, + "Setting this option to true will add the entire thrift objects to the Metastore notifications."), EVENT_DB_LISTENER_TTL("metastore.event.db.listener.timetolive", "hive.metastore.event.db.listener.timetolive", 86400, TimeUnit.SECONDS, "time after which events will be removed from the database listener queue"), diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java index bbc01c1a5e4513717f2b609fda0a7fb7bc238ac0..c076efb820d7a7f01417ba4ea4c51b3975e69368 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java @@ -20,6 +20,8 @@ import org.apache.hadoop.hive.metastore.api.Table; +import static org.apache.hadoop.hive.metastore.messaging.MessageBuilder.ADD_THRIFT_OBJECT; + public abstract class AlterTableMessage extends EventMessage { protected AlterTableMessage() { @@ -39,18 +41,20 @@ protected AlterTableMessage() { @Override public EventMessage checkValid() { if (getTable() == null) throw new IllegalStateException("Table name unset."); - try { - if (getTableObjAfter() == null){ - throw new IllegalStateException("Table object(after) not set."); - } - if (getTableObjBefore() == null){ - throw new IllegalStateException("Table object(before) not set."); - } - } catch (Exception e) { - if (! (e instanceof IllegalStateException)){ - throw new IllegalStateException("Event not set up correctly",e); - } else { - throw (IllegalStateException) e; + if (ADD_THRIFT_OBJECT) { + try { + if (getTableObjAfter() == null) { + throw new IllegalStateException("Table object(after) not set."); + } + if (getTableObjBefore() == null) { + throw new IllegalStateException("Table object(before) not set."); + } + } catch (Exception e) { + if (!(e instanceof IllegalStateException)) { + throw new IllegalStateException("Event not set up correctly", e); + } else { + throw (IllegalStateException) e; + } } } return super.checkValid(); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java index 54781436987b32ce4ee406589672358a42160589..d4768a41bd3a1c5cfa20be5b1b0fc87109539beb 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java @@ -21,6 +21,8 @@ import org.apache.hadoop.hive.metastore.api.Function; +import static org.apache.hadoop.hive.metastore.messaging.MessageBuilder.ADD_THRIFT_OBJECT; + public abstract class CreateFunctionMessage extends EventMessage { protected CreateFunctionMessage() { @@ -31,14 +33,16 @@ protected CreateFunctionMessage() { @Override public EventMessage checkValid() { - try { - if (getFunctionObj() == null) - throw new IllegalStateException("Function object unset."); - } catch (Exception e) { - if (! (e instanceof IllegalStateException)){ - throw new IllegalStateException("Event not set up correctly", e); - } else { - throw (IllegalStateException) e; + if (ADD_THRIFT_OBJECT) { + try { + if (getFunctionObj() == null) + throw new IllegalStateException("Function object unset."); + } catch (Exception e) { + if (!(e instanceof IllegalStateException)) { + throw new IllegalStateException("Event not set up correctly", e); + } else { + throw (IllegalStateException) e; + } } } return super.checkValid(); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java index 6add6c2e45167e4d71c6e10cea68d017d06d37c0..bb02466c33aeb25e47f1a7727e4fa9120beed023 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java @@ -135,6 +135,9 @@ private static final String MS_SERVICE_PRINCIPAL = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.KERBEROS_PRINCIPAL, ""); + public static final boolean ADD_THRIFT_OBJECT = + MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.EVENT_NOTIFICATIONS_ADD_THRIFT_OBJECTS); + private static volatile MessageBuilder instance; private static final Object lock = new Object(); @@ -315,38 +318,59 @@ private long now() { } public static String createPrimaryKeyObjJson(SQLPrimaryKey primaryKeyObj) throws TException { + if (!ADD_THRIFT_OBJECT) { + return null; + } TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.toString(primaryKeyObj, "UTF-8"); } public static String createForeignKeyObjJson(SQLForeignKey foreignKeyObj) throws TException { + if (!ADD_THRIFT_OBJECT) { + return null; + } TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.toString(foreignKeyObj, "UTF-8"); } public static String createUniqueConstraintObjJson(SQLUniqueConstraint uniqueConstraintObj) throws TException { + if (!ADD_THRIFT_OBJECT) { + return null; + } TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.toString(uniqueConstraintObj, "UTF-8"); } public static String createNotNullConstraintObjJson(SQLNotNullConstraint notNullConstaintObj) throws TException { + if (!ADD_THRIFT_OBJECT) { + return null; + } TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.toString(notNullConstaintObj, "UTF-8"); } public static String createDatabaseObjJson(Database dbObj) throws TException { + if (!ADD_THRIFT_OBJECT) { + return null; + } TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.toString(dbObj, "UTF-8"); } public static String createCatalogObjJson(Catalog catObj) throws TException { + if (!ADD_THRIFT_OBJECT) { + return null; + } TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.toString(catObj, "UTF-8"); } public static String createTableObjJson(Table tableObj) throws TException { + if (!ADD_THRIFT_OBJECT) { + return null; + } //Note: The parameters of the Table object will be removed in the filter if it matches // any pattern provided through EVENT_NOTIFICATION_PARAMETERS_EXCLUDE_PATTERNS filterMapkeys(tableObj.getParameters(), paramsFilter); @@ -355,6 +379,9 @@ public static String createTableObjJson(Table tableObj) throws TException { } public static String createPartitionObjJson(Partition partitionObj) throws TException { + if (!ADD_THRIFT_OBJECT) { + return null; + } //Note: The parameters of the Partition object will be removed in the filter if it matches // any pattern provided through EVENT_NOTIFICATION_PARAMETERS_EXCLUDE_PATTERNS filterMapkeys(partitionObj.getParameters(), paramsFilter); @@ -363,6 +390,9 @@ public static String createPartitionObjJson(Partition partitionObj) throws TExce } public static String createFunctionObjJson(Function functionObj) throws TException { + if (!ADD_THRIFT_OBJECT) { + return null; + } TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.toString(functionObj, "UTF-8"); }