From e5d109e48f0fbe51d03e1294192a8cbd38fbf8d0 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Thu, 21 Jan 2016 17:39:27 -0800 Subject: [PATCH] HIVE-12908 : Improve dynamic partition loading III --- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 55 ++++++++-------------- 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 0bab769..67289f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -84,7 +84,6 @@ import org.apache.hadoop.hive.metastore.api.HiveObjectType; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -1478,11 +1477,11 @@ public Partition loadPartition(Path loadPath, Table tbl, newPartPath = oldPartPath; } List newFiles = null; - if (replace) { + if (replace || (oldPart == null && !isAcid)) { Hive.replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(), isSrcLocal); } else { - if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) { + if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPart != null) { newFiles = new ArrayList<>(); } @@ -1492,7 +1491,7 @@ public Partition loadPartition(Path loadPath, Table tbl, Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath); alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString()); validatePartition(newTPart); - if (oldPart != null && null != newFiles) { + if (null != newFiles) { fireInsertEvent(tbl, partSpec, newFiles); } @@ -2594,21 +2593,20 @@ public PrincipalPrivilegeSet get_privilege_set(HiveObjectType objectType, continue; } - // Strip off the file type, if any so we don't make: - // 000000_0.gz -> 000000_0.gz_copy_1 - String name = itemSource.getName(); - String filetype; - int index = name.lastIndexOf('.'); - if (index >= 0) { - filetype = name.substring(index); - name = name.substring(0, index); - } else { - filetype = ""; - } - Path itemDest = new Path(destf, itemSource.getName()); if (!replace) { + // Strip off the file type, if any so we don't make: + // 000000_0.gz -> 000000_0.gz_copy_1 + String name = itemSource.getName(); + String filetype; + int index = name.lastIndexOf('.'); + if (index >= 0) { + filetype = name.substring(index); + name = name.substring(0, index); + } else { + filetype = ""; + } // It's possible that the file we're copying may have the same // relative name as an existing file in the "destf" directory. // So let's make a quick check to see if we can rename any @@ -2990,20 +2988,6 @@ protected static void replaceFiles(Path tablePath, Path srcf, Path destf, Path o FileSystem destFs = destf.getFileSystem(conf); // check if srcf contains nested sub-directories - FileStatus[] srcs; - FileSystem srcFs; - try { - srcFs = srcf.getFileSystem(conf); - srcs = srcFs.globStatus(srcf); - } catch (IOException e) { - throw new HiveException("Getting globStatus " + srcf.toString(), e); - } - if (srcs == null) { - LOG.info("No sources specified to move: " + srcf); - return; - } - List> result = checkPaths(conf, destFs, srcs, srcFs, destf, true); - if (oldPath != null) { boolean oldPathDeleted = false; boolean isOldPathUnderDestf = false; @@ -3051,12 +3035,11 @@ protected static void replaceFiles(Path tablePath, Path srcf, Path destf, Path o // directory if it is the root of an HDFS encryption zone. // 2. srcs must be a list of files -- ensured by LoadSemanticAnalyzer // in both cases, we move the file under destf - for (List sdpairs : result) { - for (Path[] sdpair : sdpairs) { - if (!moveFile(conf, sdpair[0], sdpair[1], true, isSrcLocal)) { - throw new IOException("Error moving: " + sdpair[0] + " into: " + sdpair[1]); - } - } + if (isSrcLocal && srcf.getFileSystem(conf).isFile(srcf)) { + destf = new Path(destf, srcf.getName()); + } + if (!moveFile(conf, srcf, destf, true, isSrcLocal)) { + throw new IOException("Error moving: " + srcf + " into: " + destf); } } catch (IOException e) { throw new HiveException(e.getMessage(), e); -- 1.7.12.4 (Apple Git-37)