From f8bfb6763e752012abbb1eae5c30cb868e704a2b Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Thu, 21 Jan 2016 17:39:27 -0800 Subject: [PATCH] HIVE-12908 : Improve dynamic partition loading III --- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 49 +++++++--------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index efb50b2..fcdddce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1427,7 +1427,7 @@ public void loadPartition(Path loadPath, String tableName, * @param isSrcLocal * If the source directory is LOCAL * @param isAcid true if this is an ACID operation - * @throws JSONException + * @throws JSONException */ public Partition loadPartition(Path loadPath, Table tbl, Map partSpec, boolean replace, @@ -1622,7 +1622,7 @@ private void constructOneLBLocationMap(FileStatus fSta, * @param txnId txnId, can be 0 unless isAcid == true * @return partition map details (PartitionSpec and Partition) * @throws HiveException - * @throws JSONException + * @throws JSONException */ public Map, Partition> loadDynamicPartitions(Path loadPath, String tableName, Map partSpec, boolean replace, @@ -2581,21 +2581,20 @@ public PrincipalPrivilegeSet get_privilege_set(HiveObjectType objectType, continue; } - // Strip off the file type, if any so we don't make: - // 000000_0.gz -> 000000_0.gz_copy_1 - String name = itemSource.getName(); - String filetype; - int index = name.lastIndexOf('.'); - if (index >= 0) { - filetype = name.substring(index); - name = name.substring(0, index); - } else { - filetype = ""; - } - Path itemDest = new Path(destf, itemSource.getName()); if (!replace) { + // Strip off the file type, if any so we don't make: + // 000000_0.gz -> 000000_0.gz_copy_1 + String name = itemSource.getName(); + String filetype; + int index = name.lastIndexOf('.'); + if (index >= 0) { + filetype = name.substring(index); + name = name.substring(0, index); + } else { + filetype = ""; + } // It's possible that the file we're copying may have the same // relative name as an existing file in the "destf" directory. // So let's make a quick check to see if we can rename any @@ -2977,20 +2976,6 @@ protected static void replaceFiles(Path tablePath, Path srcf, Path destf, Path o FileSystem destFs = destf.getFileSystem(conf); // check if srcf contains nested sub-directories - FileStatus[] srcs; - FileSystem srcFs; - try { - srcFs = srcf.getFileSystem(conf); - srcs = srcFs.globStatus(srcf); - } catch (IOException e) { - throw new HiveException("Getting globStatus " + srcf.toString(), e); - } - if (srcs == null) { - LOG.info("No sources specified to move: " + srcf); - return; - } - List> result = checkPaths(conf, destFs, srcs, srcFs, destf, true); - if (oldPath != null) { boolean oldPathDeleted = false; boolean isOldPathUnderDestf = false; @@ -3038,12 +3023,8 @@ protected static void replaceFiles(Path tablePath, Path srcf, Path destf, Path o // directory if it is the root of an HDFS encryption zone. // 2. srcs must be a list of files -- ensured by LoadSemanticAnalyzer // in both cases, we move the file under destf - for (List sdpairs : result) { - for (Path[] sdpair : sdpairs) { - if (!moveFile(conf, sdpair[0], sdpair[1], true, isSrcLocal)) { - throw new IOException("Error moving: " + sdpair[0] + " into: " + sdpair[1]); - } - } + if (!moveFile(conf, srcf, destf, true, isSrcLocal)) { + throw new IOException("Error moving: " + srcf + " into: " + destf); } } catch (IOException e) { throw new HiveException(e.getMessage(), e); -- 1.7.12.4 (Apple Git-37)