From b2a17ffc1474a886d570f5720efb409e4d6d6406 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Thu, 21 Jan 2016 17:39:27 -0800 Subject: [PATCH] HIVE-12908 : Improve dynamic partition loading III --- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 49 +++++++--------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 0bab769..225fb6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -84,7 +84,6 @@ import org.apache.hadoop.hive.metastore.api.HiveObjectType; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -2594,21 +2593,20 @@ public PrincipalPrivilegeSet get_privilege_set(HiveObjectType objectType, continue; } - // Strip off the file type, if any so we don't make: - // 000000_0.gz -> 000000_0.gz_copy_1 - String name = itemSource.getName(); - String filetype; - int index = name.lastIndexOf('.'); - if (index >= 0) { - filetype = name.substring(index); - name = name.substring(0, index); - } else { - filetype = ""; - } - Path itemDest = new Path(destf, itemSource.getName()); if (!replace) { + // Strip off the file type, if any so we don't make: + // 000000_0.gz -> 000000_0.gz_copy_1 + String name = itemSource.getName(); + String filetype; + int index = name.lastIndexOf('.'); + if (index >= 0) { + filetype = name.substring(index); + name = name.substring(0, index); + } else { + filetype = ""; + } // It's possible that the file we're copying may have the same // relative name as an existing file in the "destf" directory. // So let's make a quick check to see if we can rename any @@ -2990,20 +2988,6 @@ protected static void replaceFiles(Path tablePath, Path srcf, Path destf, Path o FileSystem destFs = destf.getFileSystem(conf); // check if srcf contains nested sub-directories - FileStatus[] srcs; - FileSystem srcFs; - try { - srcFs = srcf.getFileSystem(conf); - srcs = srcFs.globStatus(srcf); - } catch (IOException e) { - throw new HiveException("Getting globStatus " + srcf.toString(), e); - } - if (srcs == null) { - LOG.info("No sources specified to move: " + srcf); - return; - } - List> result = checkPaths(conf, destFs, srcs, srcFs, destf, true); - if (oldPath != null) { boolean oldPathDeleted = false; boolean isOldPathUnderDestf = false; @@ -3051,12 +3035,11 @@ protected static void replaceFiles(Path tablePath, Path srcf, Path destf, Path o // directory if it is the root of an HDFS encryption zone. // 2. srcs must be a list of files -- ensured by LoadSemanticAnalyzer // in both cases, we move the file under destf - for (List sdpairs : result) { - for (Path[] sdpair : sdpairs) { - if (!moveFile(conf, sdpair[0], sdpair[1], true, isSrcLocal)) { - throw new IOException("Error moving: " + sdpair[0] + " into: " + sdpair[1]); - } - } + if (isSrcLocal && srcf.getFileSystem(conf).isFile(srcf)) { + destf = new Path(destf, srcf.getName()); + } + if (!moveFile(conf, srcf, destf, true, isSrcLocal)) { + throw new IOException("Error moving: " + srcf + " into: " + destf); } } catch (IOException e) { throw new HiveException(e.getMessage(), e); -- 1.7.12.4 (Apple Git-37)