diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
index 6a5915d94e..4ff732d081 100644
--- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
+++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
@@ -74,6 +74,7 @@
protected static boolean isThriftClient = true;
private static final String CAPABILITIES_KEY = "OBJCAPABILITIES";
+ private static final String DATABASE_WAREHOUSE_SUFFIX = ".db";
@Before
public void setUp() throws Exception {
@@ -89,6 +90,7 @@ public void setUp() throws Exception {
MetastoreConf.setVar(conf, ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS,
"org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer");
MetastoreConf.setBoolVar(conf, ConfVars.HIVE_IN_TEST, false);
+ MetastoreConf.setVar(conf, ConfVars.WAREHOUSE, wh.getCanonicalPath());
MetastoreConf.setVar(conf, ConfVars.WAREHOUSE_EXTERNAL, ext_wh.getCanonicalPath());
client = new HiveMetaStoreClient(conf);
}
@@ -1174,12 +1176,15 @@ public void testTransformerAlterTable() throws Exception {
String tableLocation = tbl2.getSd().getLocation();
int idx = (tableLocation.indexOf(":") > 0) ? tableLocation.indexOf(":") : 0;
tableLocation = tableLocation.substring(idx+1);
+ String expectedPath = ext_wh.getAbsolutePath().concat(File.separator).concat(dbName).concat(DATABASE_WAREHOUSE_SUFFIX)
+ .concat(File.separator).concat(tblName);
+ assertEquals("Table location", expectedPath, tableLocation);
String newLocation = wh.getAbsolutePath().concat(File.separator).concat(dbName).concat(File.separator)
.concat(tblName);
- table.getSd().setLocation(newLocation);
+ tbl2.getSd().setLocation(newLocation);
try {
- client.alter_table(dbName, tblName, table);
+ client.alter_table(dbName, tblName, tbl2);
fail("alter_table expected to fail due to location:" + newLocation);
} catch (Exception e) {
e.printStackTrace();
diff --git standalone-metastore/metastore-server/pom.xml standalone-metastore/metastore-server/pom.xml
index 740b3ab452..7645a98255 100644
--- standalone-metastore/metastore-server/pom.xml
+++ standalone-metastore/metastore-server/pom.xml
@@ -350,6 +350,7 @@
false
${test.tmp.dir}
${test.warehouse.scheme}${test.warehouse.dir}
+ ${test.warehouse.scheme}${test.warehouse.external.dir}
${log4j.conf.dir}
@@ -417,9 +418,11 @@
+
+
@@ -571,8 +574,8 @@
false
${test.tmp.dir}
- ${test.warehouse.scheme}${test.warehouse.dir}
-
+ ${test.warehouse.scheme}${test.warehouse.dir}
+ ${test.warehouse.scheme}${test.warehouse.external.dir}
${log4j.conf.dir}
diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
index f6043cef0a..386c5a4720 100644
--- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
+++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
@@ -557,7 +557,7 @@ public Table transformCreateTable(Table table, List processorCapabilitie
params.remove(TABLE_IS_TRANSACTIONAL);
params.remove(TABLE_TRANSACTIONAL_PROPERTIES);
params.put("EXTERNAL", "TRUE");
- params.put(EXTERNAL_TABLE_PURGE, "TRUE");
+ params.put(EXTERNAL_TABLE_PURGE, "true");
params.put("TRANSLATED_TO_EXTERNAL", "TRUE");
newTable.setParameters(params);
LOG.info("Modified table params are:" + params.toString());
@@ -595,14 +595,16 @@ public Table transformCreateTable(Table table, List processorCapabilitie
} else if (TableType.EXTERNAL_TABLE.name().equals(tableType)) {
LOG.info("Table to be created is of type " + tableType + " but not " + TableType.MANAGED_TABLE.toString());
String tableLocation = table.getSd().getLocation();
- String externalWHRoot = hmsHandler.getWh().getWhRootExternal().toString();
+ Path whRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRoot());
- if (tableLocation != null && !tableLocation.startsWith(externalWHRoot)) {
- throw new MetaException(
- "An external table's location needs to be under the external warehouse root directory," + "table:"
- + table.getTableName() + ",location:" + tableLocation + ",Hive warehouse:" + externalWHRoot);
+ if (tableLocation != null) {
+ Path tablePath = Path.getPathWithoutSchemeAndAuthority(new Path(tableLocation));
+ if (tablePath.toString().startsWith(whRootPath.toString())) {
+ throw new MetaException(
+ "An external table's location should not be located within managed warehouse root directory," + "table:"
+ + table.getTableName() + ",location:" + tablePath + ",Hive managed warehouse:" + whRootPath);
+ }
}
-
}
LOG.info("Transformer returning table:" + newTable.toString());
return newTable;
@@ -613,25 +615,22 @@ public Table transformAlterTable(Table table, List processorCapabilities
LOG.info("Starting translation for Alter table for processor " + processorId + " with " + processorCapabilities
+ " on table " + table.getTableName());
String tableType = table.getTableType();
+ Path tableLocation = Path.getPathWithoutSchemeAndAuthority(new Path(table.getSd().getLocation()));
+ Path whRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRoot());
if (TableType.MANAGED_TABLE.name().equals(tableType)) {
LOG.debug("Table is a MANAGED_TABLE");
- Path tableLocation = Path.getPathWithoutSchemeAndAuthority(new Path(table.getSd().getLocation()));
- Path whRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRoot());
- if (!tableLocation.toString().startsWith(whRootPath.toString())) {
+ if (tableLocation != null && !tableLocation.toString().startsWith(whRootPath.toString())) {
throw new MetaException(
"A managed table's location needs to be under the hive warehouse root directory," + "table:"
+ table.getTableName() + ",location:" + tableLocation + ",Hive warehouse:" + whRootPath);
}
} else if (TableType.EXTERNAL_TABLE.name().equals(tableType)) {
- LOG.debug("Table is a EXTERNAL TABLE");
- Path tableLocation = Path.getPathWithoutSchemeAndAuthority(new Path(table.getSd().getLocation()));
- Path externalWHRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRootExternal());
-
- if (tableLocation != null && !tableLocation.toString().startsWith(externalWHRootPath.toString())) {
+ LOG.debug("Table is a EXTERNAL TABLE:tableLocation=" + tableLocation.toString() + ",whroot=" + whRootPath.toString());
+ if (tableLocation != null && tableLocation.toString().startsWith(whRootPath.toString())) {
throw new MetaException(
- "An external table's location needs to be under the external warehouse root directory," + "table:"
- + table.getTableName() + ",location:" + tableLocation + ",Hive external warehouse:" + externalWHRootPath);
+ "An external table's location should not be located within managed warehouse root directory," + "table:"
+ + table.getTableName() + ",location:" + tableLocation + ",Hive managed warehouse:" + whRootPath);
}
}
LOG.debug("Transformer returning table:" + table.toString());
diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java
index 0df071fb19..e53720d173 100644
--- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java
+++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java
@@ -559,7 +559,7 @@ private Table createExternalTable() throws Exception {
tableParams.put("EXTERNAL", "TRUE");
Table table = createTable("test_append_part_external_table", getYearAndMonthPartCols(),
tableParams, TableType.EXTERNAL_TABLE.name(),
- metaStore.getWarehouseRoot() + "/test_append_part_external_table");
+ metaStore.getExternalWarehouseRoot() + "/test_append_part_external_table");
return table;
}
@@ -615,7 +615,7 @@ protected void verifyPartition(Partition partition, Table table, List ex
Assert.assertEquals(expectedPartValues, partition.getValues());
Assert.assertNotEquals(0, partition.getCreateTime());
Assert.assertEquals(0, partition.getLastAccessTime());
- Assert.assertEquals(1, partition.getParameters().size());
+ Assert.assertTrue("Expect atleast transient_lastDdlTime to be set in params", (partition.getParameters().size() > 0));
Assert.assertTrue(partition.getParameters().containsKey("transient_lastDdlTime"));
StorageDescriptor partitionSD = partition.getSd();
Assert.assertEquals(table.getSd().getLocation() + "/" + partitionName,
diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDropPartitions.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDropPartitions.java
index d496294788..f84867dec9 100644
--- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDropPartitions.java
+++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestDropPartitions.java
@@ -217,7 +217,7 @@ public void testDropPartitionExternalTable() throws Exception {
tableParams.put("auto.purge", "true");
createTable(tableName, getYearPartCol(), tableParams);
- String location = metaStore.getWarehouseRoot() + "/externalTable/year=2017";
+ String location = metaStore.getExternalWarehouseRoot() + "/externalTable/year=2017";
Partition partition =
createPartition(tableName, location, Lists.newArrayList("2017"), getYearPartCol(), null);
@@ -364,7 +364,7 @@ public void testDropPartitionDeleteDataAndPurgeExternalTable() throws Exception
tableParams.put("EXTERNAL", "true");
createTable(tableName, getYearPartCol(), tableParams);
- String location = metaStore.getWarehouseRoot() + "/externalTable/year=2017";
+ String location = metaStore.getExternalWarehouseRoot() + "/externalTable/year=2017";
Partition partition =
createPartition(tableName, location, Lists.newArrayList("2017"), getYearPartCol(), null);
@@ -587,9 +587,15 @@ protected void createPartitions() throws Exception {
protected Table createTable(String tableName, List partCols,
Map tableParams) throws Exception {
String type = "MANAGED_TABLE";
- if (tableParams != null)
+ String location = metaStore.getWarehouseRoot() + "/" + tableName;
+
+ if (tableParams != null) {
type = (tableParams.getOrDefault("EXTERNAL", "FALSE").equalsIgnoreCase("TRUE")) ?
"EXTERNAL_TABLE" : "MANAGED_TABLE";
+ location = (type.equalsIgnoreCase("EXTERNAL_TABLE")) ?
+ (metaStore.getExternalWarehouseRoot() + "/" + tableName) : (metaStore.getWarehouseRoot() + "/" + tableName);
+ }
+
Table table = new TableBuilder()
.setDbName(DB_NAME)
.setTableName(tableName)
@@ -597,7 +603,7 @@ protected Table createTable(String tableName, List partCols,
.addCol("test_id", "int", "test col id")
.addCol("test_value", "string", "test col value")
.setPartCols(partCols)
- .setLocation(metaStore.getWarehouseRoot() + "/" + tableName)
+ .setLocation(location)
.setTableParams(tableParams)
.create(client, metaStore.getConf());
return table;
diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java
index 709085d71f..5e9fdca35c 100644
--- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java
+++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/AbstractMetaStoreService.java
@@ -113,6 +113,16 @@ public Path getWarehouseRoot() throws MetaException {
return warehouse.getWhRoot();
}
+ /**
+ * Returns the External MetaStore Warehouse root directory name.
+ *
+ * @return The external warehouse root directory
+ * @throws MetaException IO failure
+ */
+ public Path getExternalWarehouseRoot() throws MetaException {
+ return warehouse.getWhRootExternal();
+ }
+
/**
* Check if a path exists.
*
diff --git standalone-metastore/pom.xml standalone-metastore/pom.xml
index be0add47d9..b0c4279c96 100644
--- standalone-metastore/pom.xml
+++ standalone-metastore/pom.xml
@@ -52,6 +52,7 @@
${project.basedir}/src/test/resources
${project.build.directory}/tmp
${project.build.directory}/warehouse
+ ${project.build.directory}/external
file://
1
true