commit 810b190b576ec51fcca119cec2a2939f069676dc Author: Vihang Karajgaonkar Date: Mon Apr 17 11:10:56 2017 -0700 HIVE-16287 : Alter table partition rename with location - moves partition back to hive warehouse diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 41c6b166f7d00e947abb61f6ecd21c6cc14c8ca9..52ad97c273d260ed02f27253ac323998bdf12cf6 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -116,7 +116,7 @@ public void testCustomPerms() throws Exception { Table tbl = getTable(dbName, tblName, typeName); msc.createTable(tbl); Database db = Hive.get(hcatConf).getDatabase(dbName); - Path dfsPath = clientWH.getTablePath(db, tblName); + Path dfsPath = clientWH.getDefaultTablePath(db, tblName); cleanupTbl(dbName, tblName, typeName); // Next user did specify perms. @@ -126,7 +126,7 @@ public void testCustomPerms() throws Exception { assertTrue(e instanceof ExitException); assertEquals(((ExitException) e).getStatus(), 0); } - dfsPath = clientWH.getTablePath(db, tblName); + dfsPath = clientWH.getDefaultTablePath(db, tblName); assertTrue(dfsPath.getFileSystem(hcatConf).getFileStatus(dfsPath).getPermission().equals(FsPermission.valueOf("drwx-wx---"))); cleanupTbl(dbName, tblName, typeName); @@ -141,7 +141,7 @@ public void testCustomPerms() throws Exception { assertTrue(me instanceof ExitException); } // No physical dir gets created. - dfsPath = clientWH.getTablePath(db, tblName); + dfsPath = clientWH.getDefaultTablePath(db, tblName); try { dfsPath.getFileSystem(hcatConf).getFileStatus(dfsPath); assert false; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index c059f5e73f6447a6073198368095b387c321607e..a32f03df9eb9181e286cea1dbe095af1d2ef5d82 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -397,8 +397,9 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String msdb.alterPartition(dbname, name, part_vals, new_part); } else { try { - destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name), - Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues())); + // if tbl location is available use it + // else derive the tbl location from database location + destPath = wh.getPartitionPath(msdb.getDatabase(dbname), tbl, new_part.getValues()); destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation())); } catch (NoSuchObjectException e) { LOG.debug(e); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 7a2333decf72f8ece9dfd8bea55f8d347899d819..7baa0e696eb97c54a0822901ef70eabb45d42fdd 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1386,7 +1386,7 @@ private void create_table_core(final RawStore ms, final Table tbl, if (!TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) { if (tbl.getSd().getLocation() == null || tbl.getSd().getLocation().isEmpty()) { - tblPath = wh.getTablePath( + tblPath = wh.getDefaultTablePath( ms.getDatabase(tbl.getDbName()), tbl.getTableName()); } else { if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java index 011d4d8081047d226eb0172dbe4975bce53b3957..a8ef85ec3eb1f7cf52c127ef6a096c2f5201c581 100755 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -179,7 +179,15 @@ public Path getDefaultDatabasePath(String dbName) throws MetaException { return new Path(getWhRoot(), dbName.toLowerCase() + DATABASE_WAREHOUSE_SUFFIX); } - public Path getTablePath(Database db, String tableName) + /** + * Returns the default location of the table path using the parent database's location + * + * @param db Database where the table is created + * @param tableName table name + * @return + * @throws MetaException + */ + public Path getDefaultTablePath(Database db, String tableName) throws MetaException { return getDnsPath(new Path(getDatabasePath(db), tableName.toLowerCase())); } @@ -461,7 +469,7 @@ public static void makeSpecFromName(Map partSpec, Path currPath) public Path getPartitionPath(Database db, String tableName, LinkedHashMap pm) throws MetaException { - return new Path(getTablePath(db, tableName), makePartPath(pm)); + return new Path(getDefaultTablePath(db, tableName), makePartPath(pm)); } public Path getPartitionPath(Path tblPath, LinkedHashMap pm) @@ -469,6 +477,65 @@ public Path getPartitionPath(Path tblPath, LinkedHashMap pm) return new Path(tblPath, makePartPath(pm)); } + /** + * Returns the default partition path of a table within a given database and partition key value + * pairs. It uses the database location and appends it the table name and the partition key,value + * pairs to create the Path for the partition directory + * + * @param db - parent database which is used to get the base location of the partition directory + * @param tableName - table name for the partitions + * @param pm - Partition key value pairs + * @return + * @throws MetaException + */ + public Path getDefaultPartitionPath(Database db, String tableName, Map pm) + throws MetaException { + return getPartitionPath(getDefaultTablePath(db, tableName), pm); + } + + /** + * Returns the path object for the given partition key-value pairs and the base location + * + * @param tblPath - the base location for the partitions. Typically the table location + * @param pm - Partition key value pairs + * @return + * @throws MetaException + */ + public Path getPartitionPath(Path tblPath, Map pm) throws MetaException { + return new Path(tblPath, makePartPath(pm)); + } + + /** + * Given a database, a table and the partition key value pairs this method returns the Path object + * corresponding to the partition key value pairs. It uses the table location if available else + * uses the database location for constructing the path corresponding to the partition key-value + * pairs + * + * @param db - Parent database of the given table + * @param table - Table for which the partition key-values are given + * @param vals - List of values for the partition keys + * @return Path corresponding to the partition key-value pairs + * @throws MetaException + */ + public Path getPartitionPath(Database db, Table table, List vals) throws MetaException { + List partKeys = table.getPartitionKeys(); + if (partKeys == null || (partKeys.size() != vals.size())) { + throw new MetaException("Invalid number of partition keys found for " + table.getTableName()); + } + Map pm = new LinkedHashMap<>(vals.size()); + int i = 0; + for (FieldSchema key : partKeys) { + pm.put(key.getName(), vals.get(i)); + i++; + } + + if (table.getSd().getLocation() != null) { + return getPartitionPath(getDnsPath(new Path(table.getSd().getLocation())), pm); + } else { + return getDefaultPartitionPath(db, table.getTableName(), pm); + } + } + public boolean isDir(Path f) throws MetaException { FileSystem fs = null; try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 88c4b95d91e9318b2a85a47b80be537c575c6c86..5a761b2d5a4811417e45aa24ce63baa363b9025f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -423,7 +423,7 @@ private void fixLocationInPartSpec( } else { Database parentDb = db.getDatabase(tblDesc.getDatabaseName()); tgtPath = new Path( - wh.getTablePath( parentDb, tblDesc.getTableName()), + wh.getDefaultTablePath( parentDb, tblDesc.getTableName()), Warehouse.makePartPath(partSpec.getPartSpec())); } } else { @@ -738,7 +738,7 @@ private void createRegularImportTasks( if (tblDesc.getLocation() != null) { tablePath = new Path(tblDesc.getLocation()); } else { - tablePath = wh.getTablePath(parentDb, tblDesc.getTableName()); + tablePath = wh.getDefaultTablePath(parentDb, tblDesc.getTableName()); } checkTargetLocationEmpty(fs, tablePath, replicationSpec); t.addDependentTask(loadTable(fromURI, table, false)); @@ -781,7 +781,7 @@ private void createReplImportTasks( throw new SemanticException(ErrorMsg.DATABASE_NOT_EXISTS.getMsg(tblDesc.getDatabaseName())); } if (tblDesc.getLocation() == null) { - tblDesc.setLocation(wh.getTablePath(parentDb, tblDesc.getTableName()).toString()); + tblDesc.setLocation(wh.getDefaultTablePath(parentDb, tblDesc.getTableName()).toString()); } /* Note: In the following section, Metadata-only import handling logic is diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index ce7aeb8edcebf0132557ccf9e41999a332870719..ff4f93acb5b7d7b6a929a6e31a9d8fbc660058ad 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -183,7 +183,7 @@ public void compile(final ParseContext pCtx, final List