From 1ac579fe305d2aceb1f175af20993625bca6ddbc Mon Sep 17 00:00:00 2001 From: Alexander Kolbasov Date: Wed, 15 Aug 2018 14:28:38 -0700 Subject: [PATCH 1/1] HIVE-19253: HMS ignores tableType property for external tables --- .../apache/hadoop/hive/metastore/ObjectStore.java | 17 +++----- .../hive/metastore/utils/MetaStoreUtils.java | 3 ++ .../hadoop/hive/metastore/TestObjectStore.java | 48 ++++++++++++++++++++++ 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 8e2f94eb69..3c5246c963 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1954,19 +1954,12 @@ private MTable convertToMTable(Table tbl) throws InvalidObjectException, DatabaseName.getQualified(catName, tbl.getDbName()) + " doesn't exist."); } - // If the table has property EXTERNAL set, update table type - // accordingly + // If the table type is set as MANAGED, but the table has property + // EXTERNAL, change the table type to EXTERNAL as well. String tableType = tbl.getTableType(); - boolean isExternal = Boolean.parseBoolean(tbl.getParameters().get("EXTERNAL")); - if (TableType.MANAGED_TABLE.toString().equals(tableType)) { - if (isExternal) { - tableType = TableType.EXTERNAL_TABLE.toString(); - } - } - if (TableType.EXTERNAL_TABLE.toString().equals(tableType)) { - if (!isExternal) { - tableType = TableType.MANAGED_TABLE.toString(); - } + if (TableType.MANAGED_TABLE.toString().equals(tableType) && + Boolean.parseBoolean(tbl.getParameters().get("EXTERNAL"))) { + tableType = TableType.EXTERNAL_TABLE.toString(); } PrincipalType ownerPrincipalType = tbl.getOwnerType(); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 5233bee592..e3b8eaa6c7 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -210,6 +210,9 @@ public static boolean isExternalTable(Table table) { if (table == null) { return false; } + if (table.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) { + return true; + } Map params = table.getParameters(); if (params == null) { return false; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index b74c3048fa..b3ba216336 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; import org.apache.hadoop.hive.metastore.model.MNotificationLog; import org.apache.hadoop.hive.metastore.model.MNotificationNextId; +import org.apache.hadoop.hive.metastore.model.MTable; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; @@ -811,6 +812,53 @@ public void testNotificationOps() throws InterruptedException, MetaException { Assert.assertEquals(0, eventResponse.getEventsSize()); } + /** + * Verify that table type is set correctly based on input table properties. + * Two things are verified: + *
    + *
  1. When EXTERNAL property is set to true, table type should be external
  2. + *
  3. When table type is set to external it should remain external + *
+ * @throws Exception + */ + @Test + public void testExternalTable() throws Exception { + String t1Name = TABLE1 + "t1"; + String t2Name = TABLE1 + "t2"; + Database db1 = new DatabaseBuilder() + .setName(DB1) + .setDescription("description") + .setLocation("locationurl") + .build(conf); + objectStore.createDatabase(db1); + StorageDescriptor sd1 = + new StorageDescriptor(ImmutableList.of(new FieldSchema("pk_col", "double", null)), + "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), + null, null, null); + HashMap params = new HashMap<>(); + params.put("EXTERNAL", "true"); + Table tbl1 = + new Table(t1Name, DB1, "owner", 1, 2, 3, + sd1, null, params, null, null, + TableType.MANAGED_TABLE.toString()); + objectStore.createTable(tbl1); + params.put("EXTERNAL", "false"); + Table tbl2 = + new Table(t2Name, DB1, "owner", 1, 2, 3, sd1, + null, params, null, null, + TableType.EXTERNAL_TABLE.toString()); + objectStore.createTable(tbl2); + Table tbl3 = objectStore.getTable(db1.getCatalogName(), DB1, t1Name); + Table tbl4 = objectStore.getTable(db1.getCatalogName(), DB1, t2Name); + + Assert.assertEquals(TableType.EXTERNAL_TABLE.toString(), tbl3.getTableType()); + Assert.assertEquals(TableType.EXTERNAL_TABLE.toString(), tbl4.getTableType()); + + objectStore.dropTable(db1.getCatalogName(), DB1, t1Name); + objectStore.dropTable(db1.getCatalogName(), DB1, t2Name); + objectStore.dropDatabase(db1.getCatalogName(), DB1); + } + /** * Test metastore configuration property METASTORE_MAX_EVENT_RESPONSE */ -- 2.16.3