diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java index 6a5915d94e..5559103cef 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java @@ -74,21 +74,23 @@ protected static boolean isThriftClient = true; private static final String CAPABILITIES_KEY = "OBJCAPABILITIES"; + private static final String DATABASE_WAREHOUSE_SUFFIX = ".db"; @Before public void setUp() throws Exception { conf = MetastoreConf.newMetastoreConf(); wh = new File(System.getProperty("java.io.tmpdir") + File.separator + - "hive" + File.separator + "warehouse" + File.separator + "managed" + File.separator); + "hive" + File.separator + "warehouse" + File.separator + "hive" + File.separator); wh.mkdirs(); ext_wh = new File(System.getProperty("java.io.tmpdir") + File.separator + - "hive" + File.separator + "warehouse" + File.separator + "external" + File.separator); + "hive" + File.separator + "warehouse" + File.separator + "hive-external" + File.separator); ext_wh.mkdirs(); MetastoreConf.setVar(conf, ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS, "org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer"); MetastoreConf.setBoolVar(conf, ConfVars.HIVE_IN_TEST, false); + MetastoreConf.setVar(conf, ConfVars.WAREHOUSE, wh.getCanonicalPath()); MetastoreConf.setVar(conf, ConfVars.WAREHOUSE_EXTERNAL, ext_wh.getCanonicalPath()); client = new HiveMetaStoreClient(conf); } @@ -1174,12 +1176,15 @@ public void testTransformerAlterTable() throws Exception { String tableLocation = tbl2.getSd().getLocation(); int idx = (tableLocation.indexOf(":") > 0) ? tableLocation.indexOf(":") : 0; tableLocation = tableLocation.substring(idx+1); + String expectedPath = ext_wh.getAbsolutePath().concat(File.separator).concat(dbName).concat(DATABASE_WAREHOUSE_SUFFIX) + .concat(File.separator).concat(tblName); + assertEquals("Table location", expectedPath, tableLocation); String newLocation = wh.getAbsolutePath().concat(File.separator).concat(dbName).concat(File.separator) .concat(tblName); - table.getSd().setLocation(newLocation); + tbl2.getSd().setLocation(newLocation); try { - client.alter_table(dbName, tblName, table); + client.alter_table(dbName, tblName, tbl2); fail("alter_table expected to fail due to location:" + newLocation); } catch (Exception e) { e.printStackTrace(); diff --git standalone-metastore/metastore-server/pom.xml standalone-metastore/metastore-server/pom.xml index 740b3ab452..7645a98255 100644 --- standalone-metastore/metastore-server/pom.xml +++ standalone-metastore/metastore-server/pom.xml @@ -350,6 +350,7 @@ false ${test.tmp.dir} ${test.warehouse.scheme}${test.warehouse.dir} + ${test.warehouse.scheme}${test.warehouse.external.dir} ${log4j.conf.dir} @@ -417,9 +418,11 @@ + + @@ -571,8 +574,8 @@ false ${test.tmp.dir} - ${test.warehouse.scheme}${test.warehouse.dir} - + ${test.warehouse.scheme}${test.warehouse.dir} + ${test.warehouse.scheme}${test.warehouse.external.dir} ${log4j.conf.dir} diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java index f6043cef0a..62f87c5647 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.FileUtils; import java.util.ArrayList; import java.util.Arrays; @@ -557,7 +558,7 @@ public Table transformCreateTable(Table table, List processorCapabilitie params.remove(TABLE_IS_TRANSACTIONAL); params.remove(TABLE_TRANSACTIONAL_PROPERTIES); params.put("EXTERNAL", "TRUE"); - params.put(EXTERNAL_TABLE_PURGE, "TRUE"); + params.put(EXTERNAL_TABLE_PURGE, "true"); params.put("TRANSLATED_TO_EXTERNAL", "TRUE"); newTable.setParameters(params); LOG.info("Modified table params are:" + params.toString()); @@ -595,14 +596,16 @@ public Table transformCreateTable(Table table, List processorCapabilitie } else if (TableType.EXTERNAL_TABLE.name().equals(tableType)) { LOG.info("Table to be created is of type " + tableType + " but not " + TableType.MANAGED_TABLE.toString()); String tableLocation = table.getSd().getLocation(); - String externalWHRoot = hmsHandler.getWh().getWhRootExternal().toString(); + Path whRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRoot()); - if (tableLocation != null && !tableLocation.startsWith(externalWHRoot)) { - throw new MetaException( - "An external table's location needs to be under the external warehouse root directory," + "table:" - + table.getTableName() + ",location:" + tableLocation + ",Hive warehouse:" + externalWHRoot); + if (tableLocation != null) { + Path tablePath = Path.getPathWithoutSchemeAndAuthority(new Path(tableLocation)); + if (FileUtils.isSubdirectory(whRootPath.toString(), tablePath.toString())) { + throw new MetaException( + "An external table's location should not be located within managed warehouse root directory," + "table:" + + table.getTableName() + ",location:" + tablePath + ",Hive managed warehouse:" + whRootPath); + } } - } LOG.info("Transformer returning table:" + newTable.toString()); return newTable; @@ -613,25 +616,26 @@ public Table transformAlterTable(Table table, List processorCapabilities LOG.info("Starting translation for Alter table for processor " + processorId + " with " + processorCapabilities + " on table " + table.getTableName()); String tableType = table.getTableType(); + Path tableLocation = null; + Path whRootPath = null; if (TableType.MANAGED_TABLE.name().equals(tableType)) { LOG.debug("Table is a MANAGED_TABLE"); - Path tableLocation = Path.getPathWithoutSchemeAndAuthority(new Path(table.getSd().getLocation())); - Path whRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRoot()); - if (!tableLocation.toString().startsWith(whRootPath.toString())) { + tableLocation = Path.getPathWithoutSchemeAndAuthority(new Path(table.getSd().getLocation())); + whRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRoot()); + if (tableLocation != null && !FileUtils.isSubdirectory(whRootPath.toString(), tableLocation.toString())) { throw new MetaException( "A managed table's location needs to be under the hive warehouse root directory," + "table:" + table.getTableName() + ",location:" + tableLocation + ",Hive warehouse:" + whRootPath); } } else if (TableType.EXTERNAL_TABLE.name().equals(tableType)) { - LOG.debug("Table is a EXTERNAL TABLE"); - Path tableLocation = Path.getPathWithoutSchemeAndAuthority(new Path(table.getSd().getLocation())); - Path externalWHRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRootExternal()); - - if (tableLocation != null && !tableLocation.toString().startsWith(externalWHRootPath.toString())) { + tableLocation = Path.getPathWithoutSchemeAndAuthority(new Path(table.getSd().getLocation())); + whRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRoot()); + LOG.debug("Table is a EXTERNAL TABLE:tableLocation=" + tableLocation.toString() + ",whroot=" + whRootPath.toString()); + if (tableLocation != null && FileUtils.isSubdirectory(whRootPath.toString(), tableLocation.toString())) { throw new MetaException( - "An external table's location needs to be under the external warehouse root directory," + "table:" - + table.getTableName() + ",location:" + tableLocation + ",Hive external warehouse:" + externalWHRootPath); + "An external table's location should not be located within managed warehouse root directory," + "table:" + + table.getTableName() + ",location:" + tableLocation + ",Hive managed warehouse:" + whRootPath); } } LOG.debug("Transformer returning table:" + table.toString()); diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java index 4de673e156..c100b672b5 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java @@ -211,7 +211,7 @@ protected void assertPartitionUnchanged(Partition partition, List testVa expectedKVPairs.add(partCols.get(i) + "=" + testValues.get(i)); } String partPath = expectedKVPairs.stream().collect(joining("/")); - assertTrue(partition.getSd().getLocation().equals(metaStore.getWarehouseRoot() + assertTrue(partition.getSd().getLocation().equals(metaStore.getExternalWarehouseRoot() + "/testpartdb.db/testparttable/" + partPath)); assertNotEquals(NEW_CREATE_TIME, partition.getCreateTime()); assertNotEquals(NEW_CREATE_TIME, partition.getLastAccessTime()); @@ -227,7 +227,7 @@ protected void assertPartitionChanged(Partition partition, List testValu expectedKVPairs.add(partCols.get(i) + "=" + testValues.get(i)); } String partPath = expectedKVPairs.stream().collect(joining("/")); - assertTrue(partition.getSd().getLocation().equals(metaStore.getWarehouseRoot() + assertTrue(partition.getSd().getLocation().equals(metaStore.getExternalWarehouseRoot() + "/testpartdb.db/testparttable/" + partPath + "/hh=01")); assertEquals(NEW_CREATE_TIME, partition.getCreateTime()); assertEquals(NEW_CREATE_TIME, partition.getLastAccessTime()); diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java index 0df071fb19..e53720d173 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java @@ -559,7 +559,7 @@ private Table createExternalTable() throws Exception { tableParams.put("EXTERNAL", "TRUE"); Table table = createTable("test_append_part_external_table", getYearAndMonthPartCols(), tableParams, TableType.EXTERNAL_TABLE.name(), - metaStore.getWarehouseRoot() + "/test_append_part_external_table"); + metaStore.getExternalWarehouseRoot() + "/test_append_part_external_table"); return table; } @@ -615,7 +615,7 @@ protected void verifyPartition(Partition partition, Table table, List ex Assert.assertEquals(expectedPartValues, partition.getValues()); Assert.assertNotEquals(0, partition.getCreateTime()); Assert.assertEquals(0, partition.getLastAccessTime()); - Assert.assertEquals(1, partition.getParameters().size()); + Assert.assertTrue("Expect atleast transient_lastDdlTime to be set in params", (partition.getParameters().size() > 0)); Assert.assertTrue(partition.getParameters().containsKey("transient_lastDdlTime")); StorageDescriptor partitionSD = partition.getSd(); Assert.assertEquals(table.getSd().getLocation() + "/" + partitionName, diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDropPartitions.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDropPartitions.java index d496294788..f84867dec9 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDropPartitions.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDropPartitions.java @@ -217,7 +217,7 @@ public void testDropPartitionExternalTable() throws Exception { tableParams.put("auto.purge", "true"); createTable(tableName, getYearPartCol(), tableParams); - String location = metaStore.getWarehouseRoot() + "/externalTable/year=2017"; + String location = metaStore.getExternalWarehouseRoot() + "/externalTable/year=2017"; Partition partition = createPartition(tableName, location, Lists.newArrayList("2017"), getYearPartCol(), null); @@ -364,7 +364,7 @@ public void testDropPartitionDeleteDataAndPurgeExternalTable() throws Exception tableParams.put("EXTERNAL", "true"); createTable(tableName, getYearPartCol(), tableParams); - String location = metaStore.getWarehouseRoot() + "/externalTable/year=2017"; + String location = metaStore.getExternalWarehouseRoot() + "/externalTable/year=2017"; Partition partition = createPartition(tableName, location, Lists.newArrayList("2017"), getYearPartCol(), null); @@ -587,9 +587,15 @@ protected void createPartitions() throws Exception { protected Table createTable(String tableName, List partCols, Map tableParams) throws Exception { String type = "MANAGED_TABLE"; - if (tableParams != null) + String location = metaStore.getWarehouseRoot() + "/" + tableName; + + if (tableParams != null) { type = (tableParams.getOrDefault("EXTERNAL", "FALSE").equalsIgnoreCase("TRUE")) ? "EXTERNAL_TABLE" : "MANAGED_TABLE"; + location = (type.equalsIgnoreCase("EXTERNAL_TABLE")) ? + (metaStore.getExternalWarehouseRoot() + "/" + tableName) : (metaStore.getWarehouseRoot() + "/" + tableName); + } + Table table = new TableBuilder() .setDbName(DB_NAME) .setTableName(tableName) @@ -597,7 +603,7 @@ protected Table createTable(String tableName, List partCols, .addCol("test_id", "int", "test col id") .addCol("test_value", "string", "test col value") .setPartCols(partCols) - .setLocation(metaStore.getWarehouseRoot() + "/" + tableName) + .setLocation(location) .setTableParams(tableParams) .create(client, metaStore.getConf()); return table; diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java index b00886077d..09d386287f 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -230,9 +231,9 @@ public void testExchangePartitionsBetweenDBs() throws Exception { public void testExchangePartitionsCustomTableLocations() throws Exception { Table source = createTable(DB_NAME, "test_source_table_cust_loc", getYearMonthAndDayPartCols(), - metaStore.getWarehouseRoot() + "/sourceTable"); + metaStore.getExternalWarehouseRoot() + "/sourceTable"); Table dest = createTable(DB_NAME, "test_dest_table_cust_loc", getYearMonthAndDayPartCols(), - metaStore.getWarehouseRoot() + "/destTable"); + metaStore.getExternalWarehouseRoot() + "/destTable"); Partition[] parts = new Partition[2]; parts[0] = createPartition(source, Lists.newArrayList("2019", "may", "15"), null); parts[1] = createPartition(source, Lists.newArrayList("2019", "june", "14"), null); @@ -250,9 +251,9 @@ public void testExchangePartitionsCustomTableLocations() throws Exception { public void testExchangePartitionsCustomTableAndPartLocation() throws Exception { Table source = createTable(DB_NAME, "test_source_table_cust_loc", - getYearMonthAndDayPartCols(), metaStore.getWarehouseRoot() + "/sourceTable"); + getYearMonthAndDayPartCols(), metaStore.getExternalWarehouseRoot() + "/sourceTable"); Table dest = createTable(DB_NAME, "test_dest_table_cust_loc", getYearMonthAndDayPartCols(), - metaStore.getWarehouseRoot() + "/destTable"); + metaStore.getExternalWarehouseRoot() + "/destTable"); Partition[] parts = new Partition[2]; parts[0] = createPartition(source, Lists.newArrayList("2019", "may", "11"), source.getSd().getLocation() + "/2019m11"); @@ -753,9 +754,9 @@ public void testExchangePartitionBetweenDBs() throws Exception { public void testExchangePartitionCustomTableLocations() throws Exception { Table source = createTable(DB_NAME, "test_source_table_cust_loc", - getYearMonthAndDayPartCols(), metaStore.getWarehouseRoot() + "/sourceTable"); + getYearMonthAndDayPartCols(), metaStore.getExternalWarehouseRoot() + "/sourceTable"); Table dest = createTable(DB_NAME, "test_dest_table_cust_loc", getYearMonthAndDayPartCols(), - metaStore.getWarehouseRoot() + "/destTable"); + metaStore.getExternalWarehouseRoot() + "/destTable"); Partition[] parts = new Partition[2]; parts[0] = createPartition(source, Lists.newArrayList("2019", "may", "15"), null); parts[1] = createPartition(source, Lists.newArrayList("2019", "june", "14"), null); @@ -773,9 +774,9 @@ public void testExchangePartitionCustomTableLocations() throws Exception { public void testExchangePartitionCustomTableAndPartLocation() throws Exception { Table source = createTable(DB_NAME, "test_source_table_cust_loc", - getYearMonthAndDayPartCols(), metaStore.getWarehouseRoot() + "/sourceTable"); + getYearMonthAndDayPartCols(), metaStore.getExternalWarehouseRoot() + "/sourceTable"); Table dest = createTable(DB_NAME, "test_dest_table_cust_loc", getYearMonthAndDayPartCols(), - metaStore.getWarehouseRoot() + "/destTable"); + metaStore.getExternalWarehouseRoot() + "/destTable"); Partition[] parts = new Partition[2]; parts[0] = createPartition(source, Lists.newArrayList("2019", "may", "11"), source.getSd().getLocation() + "/2019m11"); @@ -1219,6 +1220,7 @@ protected Table createTable(String dbName, String tableName, List p new TableBuilder() .setDbName(dbName) .setTableName(tableName) + .setType(TableType.EXTERNAL_TABLE.name()) .setCols(cols) .setPartCols(partCols) .setLocation(location) diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java index 709085d71f..5e9fdca35c 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java @@ -113,6 +113,16 @@ public Path getWarehouseRoot() throws MetaException { return warehouse.getWhRoot(); } + /** + * Returns the External MetaStore Warehouse root directory name. + * + * @return The external warehouse root directory + * @throws MetaException IO failure + */ + public Path getExternalWarehouseRoot() throws MetaException { + return warehouse.getWhRootExternal(); + } + /** * Check if a path exists. * diff --git standalone-metastore/pom.xml standalone-metastore/pom.xml index be0add47d9..b0c4279c96 100644 --- standalone-metastore/pom.xml +++ standalone-metastore/pom.xml @@ -52,6 +52,7 @@ ${project.basedir}/src/test/resources ${project.build.directory}/tmp ${project.build.directory}/warehouse + ${project.build.directory}/external file:// 1 true