commit e47d8e85043d7fe4d990c55768737b81e4dc203e Author: Vihang Karajgaonkar Date: Wed Apr 5 11:02:28 2017 -0700 HIVE-16287 : Alter table partition rename with location - moves partition back to hive warehouse diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 8aa510fa99b287a8d8c5072cf07697b13d4f242e..66a5dd43d544c29ece588e54673de581b7625d79 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -116,7 +116,7 @@ public void testCustomPerms() throws Exception { Table tbl = getTable(dbName, tblName, typeName); msc.createTable(tbl); Database db = Hive.get(hcatConf).getDatabase(dbName); - Path dfsPath = clientWH.getTablePath(db, tblName); + Path dfsPath = clientWH.getDefaultTablePath(db, tblName); cleanupTbl(dbName, tblName, typeName); // Next user did specify perms. @@ -126,7 +126,7 @@ public void testCustomPerms() throws Exception { assertTrue(e instanceof ExitException); assertEquals(((ExitException) e).getStatus(), 0); } - dfsPath = clientWH.getTablePath(db, tblName); + dfsPath = clientWH.getDefaultTablePath(db, tblName); assertTrue(dfsPath.getFileSystem(hcatConf).getFileStatus(dfsPath).getPermission().equals(FsPermission.valueOf("drwx-wx---"))); cleanupTbl(dbName, tblName, typeName); @@ -141,7 +141,7 @@ public void testCustomPerms() throws Exception { assertTrue(me instanceof ExitException); } // No physical dir gets created. - dfsPath = clientWH.getTablePath(db, tblName); + dfsPath = clientWH.getDefaultTablePath(db, tblName); try { dfsPath.getFileSystem(hcatConf).getFileStatus(dfsPath); assert false; diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java index 1ac4d01e6fdcee4c1fc6e4e8718ccdc29222fcc5..3f9eec3e35d67e9aecbab273058e1f9bd363c029 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java @@ -151,15 +151,15 @@ public void testRecyclePartTable() throws Exception { Partition part3 = createPartition(dbName, tblName, columns, values, serdeInfo); client.add_partition(part3); - Path part1Path = new Path(warehouse.getPartitionPath(db, tblName, ImmutableMap.of("dt", "20160101")), "part"); + Path part1Path = new Path(warehouse.getDefaultPartitionPath(db, tblName, ImmutableMap.of("dt", "20160101")), "part"); createFile(part1Path, "p1"); String path1Chksum = ReplChangeManager.getChksumString(part1Path, fs); - Path part2Path = new Path(warehouse.getPartitionPath(db, tblName, ImmutableMap.of("dt", "20160102")), "part"); + Path part2Path = new Path(warehouse.getDefaultPartitionPath(db, tblName, ImmutableMap.of("dt", "20160102")), "part"); createFile(part2Path, "p2"); String path2Chksum = ReplChangeManager.getChksumString(part2Path, fs); - Path part3Path = new Path(warehouse.getPartitionPath(db, tblName, ImmutableMap.of("dt", "20160103")), "part"); + Path part3Path = new Path(warehouse.getDefaultPartitionPath(db, tblName, ImmutableMap.of("dt", "20160103")), "part"); createFile(part3Path, "p3"); String path3Chksum = ReplChangeManager.getChksumString(part3Path, fs); @@ -221,15 +221,15 @@ public void testRecycleNonPartTable() throws Exception { client.createTable(tbl); - Path filePath1 = new Path(warehouse.getTablePath(db, tblName), "part1"); + Path filePath1 = new Path(warehouse.getDefaultTablePath(db, tblName), "part1"); createFile(filePath1, "f1"); String fileChksum1 = ReplChangeManager.getChksumString(filePath1, fs); - Path filePath2 = new Path(warehouse.getTablePath(db, tblName), "part2"); + Path filePath2 = new Path(warehouse.getDefaultTablePath(db, tblName), "part2"); createFile(filePath2, "f2"); String fileChksum2 = ReplChangeManager.getChksumString(filePath2, fs); - Path filePath3 = new Path(warehouse.getTablePath(db, tblName), "part3"); + Path filePath3 = new Path(warehouse.getDefaultTablePath(db, tblName), "part3"); createFile(filePath3, "f3"); String fileChksum3 = ReplChangeManager.getChksumString(filePath3, fs); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index de704e8cd3612022b3ff65ab4f77f9ebf40a9cf4..77b3541517954a0998f55f9e94a4db3b04724b05 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -442,8 +442,9 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String msdb.alterPartition(dbname, name, part_vals, new_part); } else { try { - destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name), - Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues())); + // if tbl location is available use it + // else derive the tbl location from database location + destPath = wh.getPartitionPath(msdb.getDatabase(dbname), tbl, new_part.getValues()); destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation())); } catch (NoSuchObjectException e) { LOG.debug("Didn't find object in metastore ", e); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index a4693bd89de61f96ff72fad577fe02c3a2a0b1fa..a8fa1ef4e3e8f6a8f9ec916757d33d7562dcf97b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1418,7 +1418,7 @@ private void create_table_core(final RawStore ms, final Table tbl, if (!TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) { if (tbl.getSd().getLocation() == null || tbl.getSd().getLocation().isEmpty()) { - tblPath = wh.getTablePath( + tblPath = wh.getDefaultTablePath( ms.getDatabase(tbl.getDbName()), tbl.getTableName()); } else { if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java index f8a98e8d77f4220a710913bfd5f68668307f6832..8134ab2cff85724c3d16c60462d064003b62b89b 100755 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -151,11 +151,6 @@ public Path getWhRoot() throws MetaException { return whRoot; } - public Path getTablePath(String whRootString, String tableName) throws MetaException { - Path whRoot = getDnsPath(new Path(whRootString)); - return new Path(whRoot, MetaStoreUtils.encodeTableName(tableName.toLowerCase())); - } - public Path getDatabasePath(Database db) throws MetaException { if (db.getName().equalsIgnoreCase(DEFAULT_DATABASE_NAME)) { return getWhRoot(); @@ -170,7 +165,14 @@ public Path getDefaultDatabasePath(String dbName) throws MetaException { return new Path(getWhRoot(), dbName.toLowerCase() + DATABASE_WAREHOUSE_SUFFIX); } - public Path getTablePath(Database db, String tableName) + /** + * Returns the default location of the table path using the parent database's location + * @param db Database where the table is created + * @param tableName table name + * @return + * @throws MetaException + */ + public Path getDefaultTablePath(Database db, String tableName) throws MetaException { return getDnsPath(new Path(getDatabasePath(db), MetaStoreUtils.encodeTableName(tableName.toLowerCase()))); } @@ -444,16 +446,67 @@ public static void makeSpecFromName(Map partSpec, Path currPath) return partSpec; } - public Path getPartitionPath(Database db, String tableName, + /** + * Returns the default partition path of a table within a given database and partition key value + * pairs. It uses the database location and appends it the table name and the partition key,value + * pairs to create the Path for the partition directory + * + * @param db - parent database which is used to get the base location of the partition directory + * @param tableName - table name for the partitions + * @param pm - Partition key value pairs + * @return + * @throws MetaException + */ + public Path getDefaultPartitionPath(Database db, String tableName, Map pm) throws MetaException { - return new Path(getTablePath(db, tableName), makePartPath(pm)); + return getPartitionPath(getDefaultTablePath(db, tableName), pm); } + /** + * Returns the path object for the given partition key-value pairs and the base location + * + * @param tblPath - the base location for the partitions. Typically the table location + * @param pm - Partition key value pairs + * @return + * @throws MetaException + */ public Path getPartitionPath(Path tblPath, Map pm) throws MetaException { return new Path(tblPath, makePartPath(pm)); } + /** + * Given a database, a table and the partition key value pairs this method returns the Path object + * corresponding to the partition key value pairs. It uses the table location if available else + * uses the database location for constructing the path corresponding to the partition key-value + * pairs + * + * @param db - Parent database of the given table + * @param table - Table for which the partition key-values are given + * @param vals - List of values for the partition keys + * @return Path corresponding to the partition key-value pairs + * @throws MetaException + */ + public Path getPartitionPath(Database db, Table table, List vals) + throws MetaException { + List partKeys = table.getPartitionKeys(); + if (partKeys == null || (partKeys.size() != vals.size())) { + throw new MetaException("Invalid number of partition keys found for " + table.getTableName()); + } + Map pm = new LinkedHashMap<>(vals.size()); + int i = 0; + for (FieldSchema key : partKeys) { + pm.put(key.getName(), vals.get(i)); + i++; + } + + if (table.getSd().getLocation() != null) { + return getPartitionPath(getDnsPath(new Path(table.getSd().getLocation())), pm); + } else { + return getDefaultPartitionPath(db, table.getTableName(), pm); + } + } + public boolean isDir(Path f) throws MetaException { FileSystem fs = null; try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 245c48357bbec7dd52d8bb5485210feeb7cfcf92..ec602ae3643eba8e8a21c64f0936cdd8af63edc8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -441,7 +441,7 @@ private static void fixLocationInPartSpec( } else { Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); tgtPath = new Path( - wh.getTablePath( parentDb, tblDesc.getTableName()), + wh.getDefaultTablePath( parentDb, tblDesc.getTableName()), Warehouse.makePartPath(partSpec.getPartSpec())); } } else { @@ -768,7 +768,7 @@ private static void createRegularImportTasks( if (tblDesc.getLocation() != null) { tablePath = new Path(tblDesc.getLocation()); } else { - tablePath = wh.getTablePath(parentDb, tblDesc.getTableName()); + tablePath = wh.getDefaultTablePath(parentDb, tblDesc.getTableName()); } FileSystem tgtFs = FileSystem.get(tablePath.toUri(), x.getConf()); checkTargetLocationEmpty(tgtFs, tablePath, replicationSpec,x); @@ -821,7 +821,7 @@ private static void createReplImportTasks( } if (tblDesc.getLocation() == null) { if (!waitOnPrecursor){ - tblDesc.setLocation(wh.getTablePath(parentDb, tblDesc.getTableName()).toString()); + tblDesc.setLocation(wh.getDefaultTablePath(parentDb, tblDesc.getTableName()).toString()); } else { tblDesc.setLocation( wh.getDnsPath(new Path( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e27e953d76cadc3ab74096864c3b0124392ae7de..77fc35d36d95da7bedfbc7f15e8172c22404598e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7276,7 +7276,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) String tName = Utilities.getDbTableName(tableDesc.getTableName())[1]; try { Warehouse wh = new Warehouse(conf); - tlocation = wh.getTablePath(db.getDatabase(tableDesc.getDatabaseName()), tName); + tlocation = wh.getDefaultTablePath(db.getDatabase(tableDesc.getDatabaseName()), tName); } catch (MetaException|HiveException e) { throw new SemanticException(e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 5f9ccc8cebb6886dd4049ac7ab825574f66ddb27..7caeb78e4459258465b509081c4f17148de8e8f9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -257,7 +257,7 @@ public void compile(final ParseContext pCtx, final List