From d59bb4ec1e35f4c49a823741f1349808cfd6a626 Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Thu, 21 Sep 2017 17:38:36 -0700 Subject: [PATCH 1/1] HIVE-17402 Provide more useful information in the HMS notification messages --- .../messaging/json/JSONAddPartitionMessage.java | 16 ++++++++++ .../messaging/json/JSONAlterPartitionMessage.java | 28 +++++++++++++++++ .../messaging/json/JSONAlterTableMessage.java | 27 ++++++++++++++++ .../messaging/json/JSONCreateDatabaseMessage.java | 14 ++++++++- .../messaging/json/JSONCreateTableMessage.java | 13 ++++++++ .../messaging/json/JSONDropDatabaseMessage.java | 12 ++++++++ .../messaging/json/JSONDropPartitionMessage.java | 14 +++++++++ .../messaging/json/JSONDropTableMessage.java | 12 ++++++++ .../messaging/json/JSONMessageFactory.java | 36 +++++++++++++++++----- 9 files changed, 163 insertions(+), 9 deletions(-) diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java index 5d201f465f..ff10708ac0 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAddPartitionMessage.java @@ -22,6 +22,7 @@ import org.apache.hive.hcatalog.messaging.AddPartitionMessage; import org.codehaus.jackson.annotate.JsonProperty; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -39,6 +40,10 @@ @JsonProperty List> partitions; + /** Location for each partition */ + @JsonProperty + List locations; + /** * Default Constructor. Required for Jackson. */ @@ -51,6 +56,13 @@ public JSONAddPartitionMessage(String server, String servicePrincipal, String db public JSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, String tableType, List> partitions, Long timestamp) { + this(server, servicePrincipal, db, table, tableType, partitions, Collections.emptyList(), + timestamp); + } + + public JSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, + String tableType, List> partitions, + List locations, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; @@ -58,8 +70,12 @@ public JSONAddPartitionMessage(String server, String servicePrincipal, String db this.tableType = tableType; this.partitions = partitions; this.timestamp = timestamp; + this.locations = locations; checkValid(); } + public List getLocations() { + return locations; + } @Override public String getServer() { return server; } diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java index 7ae7d75168..61df8a5fa9 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterPartitionMessage.java @@ -39,6 +39,12 @@ @JsonProperty Long timestamp; + @JsonProperty + private String newLocation; + + @JsonProperty + private String oldLocation; + @JsonProperty Map keyValues; @@ -63,6 +69,18 @@ public JSONAlterPartitionMessage(String server, String tableType, Map keyValues, Long timestamp) { + this(server, servicePrincipal, db, table, tableType, keyValues, "", "", timestamp); + } + + public JSONAlterPartitionMessage(String server, + String servicePrincipal, + String db, + String table, + String tableType, + Map keyValues, + String oldLocation, + String newLocation, + Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; @@ -70,9 +88,19 @@ public JSONAlterPartitionMessage(String server, this.tableType = tableType; this.timestamp = timestamp; this.keyValues = keyValues; + this.oldLocation = oldLocation; + this.newLocation = newLocation; checkValid(); } + public String getNewLocation() { + return newLocation; + } + + public String getOldLocation() { + return oldLocation; + } + @Override public String getServer() { return server; diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java index f23aedf0b3..95f948432b 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterTableMessage.java @@ -34,6 +34,12 @@ @JsonProperty String server, servicePrincipal, db, table, tableType; + @JsonProperty + private String newLocation; + + @JsonProperty + private String oldLocation; + @JsonProperty Long timestamp; @@ -56,15 +62,36 @@ public JSONAlterTableMessage(String server, String table, String tableType, Long timestamp) { + this(server, servicePrincipal, db, table, tableType, "", "", timestamp); + } + + public JSONAlterTableMessage(String server, + String servicePrincipal, + String db, + String table, + String tableType, + String oldLocation, + String newLocation, + Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; this.tableType = tableType; this.timestamp = timestamp; + this.oldLocation = oldLocation; + this.newLocation= newLocation; checkValid(); } + public String getNewLocation() { + return newLocation; + } + + public String getOldLocation() { + return oldLocation; + } + @Override public String getServer() { return server; diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateDatabaseMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateDatabaseMessage.java index 6db46cf773..173de79b32 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateDatabaseMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateDatabaseMessage.java @@ -30,6 +30,9 @@ @JsonProperty String server, servicePrincipal, db; + @JsonProperty + private String location; + @JsonProperty Long timestamp; @@ -39,13 +42,23 @@ public JSONCreateDatabaseMessage() {} public JSONCreateDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp) { + this(server, servicePrincipal, db, "", timestamp); + } + + public JSONCreateDatabaseMessage(String server, String servicePrincipal, String db, + String location, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.timestamp = timestamp; + this.location = location; checkValid(); } + public String getLocation() { + return location; + } + @Override public String getDB() { return db; } @@ -67,5 +80,4 @@ public String toString() { throw new IllegalArgumentException("Could not serialize: ", exception); } } - } diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java index 327dc972de..7a938a1c95 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateTableMessage.java @@ -30,6 +30,9 @@ @JsonProperty String server, servicePrincipal, db, table, tableType; + @JsonProperty + private String location; + @JsonProperty Long timestamp; @@ -45,15 +48,25 @@ public JSONCreateTableMessage(String server, String servicePrincipal, String db, public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, String tableType, Long timestamp) { + this(server, servicePrincipal, db, table, tableType, "", timestamp); + } + + public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, + String tableType, String location, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; this.tableType = tableType; this.timestamp = timestamp; + this.location = location; checkValid(); } + public String getLocation() { + return location; + } + @Override public String getServer() { return server; } diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropDatabaseMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropDatabaseMessage.java index bfe3f631ea..811cc03e49 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropDatabaseMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropDatabaseMessage.java @@ -30,6 +30,9 @@ @JsonProperty String server, servicePrincipal, db; + @JsonProperty + private String location; + @JsonProperty Long timestamp; @@ -39,13 +42,22 @@ public JSONDropDatabaseMessage() {} public JSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp) { + this(server, servicePrincipal, db, "", timestamp); + } + + public JSONDropDatabaseMessage(String server, String servicePrincipal, String db, + String location, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.timestamp = timestamp; + this.location = location; checkValid(); } + public String getLocation() { + return location; + } @Override public String getServer() { return server; } diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java index ecdc39d059..418cf8ae26 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropPartitionMessage.java @@ -39,6 +39,10 @@ @JsonProperty List> partitions; + /** Location for each partition */ + @JsonProperty + List locations; + /** * Default Constructor. Required for Jackson. */ @@ -51,6 +55,12 @@ public JSONDropPartitionMessage(String server, String servicePrincipal, String d public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, String tableType, List> partitions, Long timestamp) { + this(server, servicePrincipal, db, table, tableType, partitions, null, timestamp); + } + + public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, + String tableType, List> partitions, + List locations, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; @@ -58,9 +68,13 @@ public JSONDropPartitionMessage(String server, String servicePrincipal, String d this.tableType = tableType; this.partitions = partitions; this.timestamp = timestamp; + this.locations = locations; checkValid(); } + public List getLocations() { + return locations; + } @Override public String getServer() { return server; } diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java index 430062544b..420d15ab72 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropTableMessage.java @@ -30,6 +30,9 @@ @JsonProperty String server, servicePrincipal, db, table, tableType; + @JsonProperty + private String location; + @JsonProperty Long timestamp; @@ -40,15 +43,24 @@ public JSONDropTableMessage() {} public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, String tableType, Long timestamp) { + this(server, servicePrincipal, db, table, tableType, "", timestamp); + } + + public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, + String tableType, String location, Long timestamp) { this.server = server; this.servicePrincipal = servicePrincipal; this.db = db; this.table = table; this.tableType = tableType; this.timestamp = timestamp; + this.location = location; checkValid(); } + public String getLocation() { + return location; + } @Override public String getTable() { return table; } diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java index 796273299f..7824134a65 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java @@ -50,6 +50,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -83,52 +84,60 @@ public String getMessageFormat() { @Override public CreateDatabaseMessage buildCreateDatabaseMessage(Database db) { return new JSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), - now()); + db.getLocationUri(), now()); } @Override public DropDatabaseMessage buildDropDatabaseMessage(Database db) { return new JSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), - now()); + db.getLocationUri(), now()); } @Override public CreateTableMessage buildCreateTableMessage(Table table) { + String tableLocation = table.getSd() != null ? table.getSd().getLocation() : ""; return new JSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), table.getTableType(), now()); + table.getTableName(), table.getTableType(), tableLocation, now()); } @Override public AlterTableMessage buildAlterTableMessage(Table before, Table after) { + String tableLocationBefore = before.getSd() != null ? before.getSd().getLocation() : ""; + String tableLocationAfter = after.getSd() != null ? after.getSd().getLocation() : ""; return new JSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), - before.getTableName(), before.getTableType(), now()); + before.getTableName(), before.getTableType(), + tableLocationBefore, tableLocationAfter, now()); } @Override public DropTableMessage buildDropTableMessage(Table table) { + String tableLocation = table.getSd() != null ? table.getSd().getLocation() : ""; return new JSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), table.getTableType(), now()); + table.getTableName(), table.getTableType(), tableLocation, now()); } @Override public AddPartitionMessage buildAddPartitionMessage(Table table, Iterator partitionsIterator) { return new JSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), table.getTableName(), table.getTableType(), - getPartitionKeyValues(table, partitionsIterator), now()); + getPartitionKeyValues(table, partitionsIterator), getPartitionLocations(partitionsIterator), + now()); } @Override public AlterPartitionMessage buildAlterPartitionMessage(Table table, Partition before, Partition after) { + String partLocationBefore = before.getSd() != null ? before.getSd().getLocation() : ""; + String partLocationAfter = after.getSd() != null ? after.getSd().getLocation() : ""; return new JSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), before.getTableName(), table.getTableType(), - getPartitionKeyValues(table,before),now()); + getPartitionKeyValues(table,before), partLocationBefore, partLocationAfter, now()); } @Override public DropPartitionMessage buildDropPartitionMessage(Table table, Iterator partitions) { return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), table.getTableName(), table.getTableType(), - getPartitionKeyValues(table, partitions), now()); + getPartitionKeyValues(table, partitions), getPartitionLocations(partitions),now()); } @Override @@ -196,6 +205,17 @@ private long now() { })); } + /** + * @param iterator partitions iterator + * @return list of partition locations for each partition provided + */ + private List getPartitionLocations(Iterator iterator) { + return Lists.newArrayList(Iterators.transform(iterator, + partition -> partition == null ? "" : + partition.getSd() == null ? "" : + partition.getSd().getLocation())); + } + static String createFunctionObjJson(Function functionObj) throws TException { TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.toString(functionObj, "UTF-8"); -- 2.14.1