diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java index 4fea863d27..466cce63e4 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java @@ -920,13 +920,36 @@ public void testCreateTable() throws Exception { Table tbl2 = client.getTable(dbName, tblName); assertEquals("Table type expected to be EXTERNAL", "EXTERNAL_TABLE", tbl2.getTableType()); - tblName = "test_create_table_mgd_woc"; + tblName = "test_create_table_mgd_wc"; type = TableType.MANAGED_TABLE; tProps.put("TBLNAME", tblName); tProps.put("TBLTYPE", TableType.MANAGED_TABLE); table_params = new StringBuilder(); table_params.append("key1=val1"); + table_params.append(";"); + table_params.append("transactional_properties=default"); tProps.put("PROPERTIES", table_params.toString()); + + List capabilities = new ArrayList<>(); + capabilities.add("HIVEFULLACIDWRITE"); + setHMSClient("TestCreateTableMGD#1", (String[])(capabilities.toArray(new String[0]))); + + table = createTableWithCapabilities(tProps); + + // retrieve the table + tbl2 = client.getTable(dbName, tblName); + assertEquals("Table type expected to be converted to EXTERNAL", "EXTERNAL_TABLE", tbl2.getTableType()); + assertNotNull("external.table.purge is expected to be non-null", tbl2.getParameters().get("external.table.purge")); + assertTrue("external.table.purge is expected to be true", + tbl2.getParameters().get("external.table.purge").equalsIgnoreCase("TRUE")); + assertTrue("Table params expected to contain original properties", tbl2.getParameters().get("key1").equals("val1")); + + resetHMSClient(); + + capabilities = new ArrayList<>(); + capabilities.add("HIVEMANAGEDINSERTWRITE"); + setHMSClient("TestCreateTableMGD#2", (String[])(capabilities.toArray(new String[0]))); + table = createTableWithCapabilities(tProps); // retrieve the table @@ -937,6 +960,8 @@ public void testCreateTable() throws Exception { tbl2.getParameters().get("external.table.purge").equalsIgnoreCase("TRUE")); assertTrue("Table params expected to contain original properties", tbl2.getParameters().get("key1").equals("val1")); + resetHMSClient(); + // Test for FULL ACID tables tblName = "test_create_table_acid_mgd_woc"; type = TableType.MANAGED_TABLE; @@ -955,7 +980,7 @@ public void testCreateTable() throws Exception { LOG.info("Create table expected to fail as ACID table cannot be created without possessing capabilities"); } - List capabilities = new ArrayList<>(); + capabilities = new ArrayList<>(); capabilities.add("CONNECTORWRITE"); setHMSClient("TestCreateTableACID#1", (String[])(capabilities.toArray(new String[0]))); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java index 7557280d2d..602394a10e 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java @@ -71,6 +71,7 @@ public static void beforeClassSetup() throws Exception { primary = new WarehouseInstance(LOG, miniDFSCluster, new HashMap() {{ put(HiveConf.ConfVars.HIVE_IN_TEST.varname, "false"); put(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false"); + put(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname, "true"); }}, "test_key"); } @@ -110,6 +111,7 @@ public void targetAndSourceHaveDifferentEncryptionZoneKeys() throws Throwable { replica .run("repl load " + replicatedDbName + " from '" + tuple.dumpLocation + "' with('hive.repl.add.raw.reserved.namespace'='true', " + + "'hive.repl.replica.external.table.base.dir'='" + replica.externalTableWarehouseRoot + "', " + "'distcp.options.pugpbx'='', 'distcp.options.skipcrccheck'='')") .run("use " + replicatedDbName) .run("repl status " + replicatedDbName) diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java index 06f3630067..e723aad326 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java @@ -543,38 +543,38 @@ public Table transformCreateTable(Table table, List processorCapabilitie params = new HashMap<>(); String tableType = newTable.getTableType(); String txnal = null; + String txn_properties = null; + boolean isInsertAcid = false; if (TableType.MANAGED_TABLE.name().equals(tableType)) { LOG.info("Table is a MANAGED_TABLE"); txnal = params.get(TABLE_IS_TRANSACTIONAL); - if (txnal == null || txnal.equalsIgnoreCase("FALSE")) { // non-ACID MANAGED TABLE - if (processorCapabilities == null || (!processorCapabilities.contains(HIVEMANAGEDINSERTWRITE) && - !processorCapabilities.contains(HIVEFULLACIDWRITE))) { - LOG.info("Converting " + newTable.getTableName() + " to EXTERNAL tableType for " + processorId); - newTable.setTableType(TableType.EXTERNAL_TABLE.toString()); - params.remove(TABLE_IS_TRANSACTIONAL); - params.remove(TABLE_TRANSACTIONAL_PROPERTIES); - params.put("EXTERNAL", "TRUE"); - params.put(EXTERNAL_TABLE_PURGE, "TRUE"); - params.put("TRANSLATED_TO_EXTERNAL", "TRUE"); - newTable.setParameters(params); - LOG.info("Modified table params are:" + params.toString()); - if (table.getSd().getLocation() == null) { - try { - Path newPath = hmsHandler.getWh().getDefaultTablePath(table.getDbName(), table.getTableName(), true); - newTable.getSd().setLocation(newPath.toString()); - LOG.info("Modified location from to " + newPath); - } catch (Exception e) { - LOG.warn("Exception determining external table location:" + e.getMessage()); - } + txn_properties = params.get(TABLE_TRANSACTIONAL_PROPERTIES); + isInsertAcid = (txn_properties != null && txn_properties.equalsIgnoreCase("insert_only")); + if ((txnal == null || txnal.equalsIgnoreCase("FALSE")) && !isInsertAcid) { // non-ACID MANAGED TABLE + LOG.info("Converting " + newTable.getTableName() + " to EXTERNAL tableType for " + processorId); + newTable.setTableType(TableType.EXTERNAL_TABLE.toString()); + params.remove(TABLE_IS_TRANSACTIONAL); + params.remove(TABLE_TRANSACTIONAL_PROPERTIES); + params.put("EXTERNAL", "TRUE"); + params.put(EXTERNAL_TABLE_PURGE, "TRUE"); + params.put("TRANSLATED_TO_EXTERNAL", "TRUE"); + newTable.setParameters(params); + LOG.info("Modified table params are:" + params.toString()); + if (table.getSd().getLocation() == null) { + try { + Path newPath = hmsHandler.getWh().getDefaultTablePath(table.getDbName(), table.getTableName(), true); + newTable.getSd().setLocation(newPath.toString()); + LOG.info("Modified location from null to " + newPath); + } catch (Exception e) { + LOG.warn("Exception determining external table location:" + e.getMessage()); } } } else { // ACID table if (processorCapabilities == null || processorCapabilities.isEmpty()) { throw new MetaException("Processor has no capabilities, cannot create an ACID table."); } - String txntype = params.get(TABLE_TRANSACTIONAL_PROPERTIES); - if (txntype != null && txntype.equalsIgnoreCase("insert_only")) { // MICRO_MANAGED Tables + if (isInsertAcid) { // MICRO_MANAGED Tables if (processorCapabilities.contains(HIVEMANAGEDINSERTWRITE)) { LOG.info("Processor has required capabilities to be able to create INSERT-only tables"); return newTable;