diff --git common/src/java/org/apache/hadoop/hive/common/FileUtils.java common/src/java/org/apache/hadoop/hive/common/FileUtils.java index ff09dd835c..ff92f61c0f 100644 --- common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -28,18 +28,12 @@ import java.nio.ByteBuffer; import java.security.AccessControlException; import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.BitSet; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; +import java.util.*; +import java.util.regex.Pattern; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; @@ -1075,4 +1069,54 @@ public static void readFully(InputStream stream, int length, ByteBuffer bb) thro bb.position(bb.position() + fullLen); } } + + public static String replaceByEmpty(String str, Pattern pattern) { + return Strings.isNullOrEmpty(str) ? null : pattern.matcher(str).replaceAll(""); + } + + public static String getFsPathByMountPath(Configuration conf, String path) { + if (Strings.isNullOrEmpty(path)) { + return null; + } + String mountPath = replaceByEmpty(path, Pattern.compile("^" + conf.get("fs.defaultFS"))); + if (Strings.isNullOrEmpty(mountPath)) { + return null; + } + Map mountTableSettings = conf.getValByRegex("^fs.viewfs.mounttable.nsX.link."); + + for (String mountPathSetting : mountTableSettings.keySet()) { + String mountPathSettingActual = replaceByEmpty(mountPathSetting, Pattern.compile("^fs.viewfs.mounttable.nsX.link.")); + if (!Strings.isNullOrEmpty(mountPathSettingActual) && + (mountPath.equals(mountPathSettingActual) + || mountPath.startsWith(mountPathSettingActual + "/"))) { + return replaceByEmpty(mountTableSettings.get(mountPathSetting), Pattern.compile("^hdfs:/")) + + mountPath.substring(mountPathSettingActual.length()); + } + } + return null; + } + + public static boolean equalsMountPointPath(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs) { + String srcMountFs = null; + String destMountFs = null; + if("viewfs".equals(srcFs.getUri().getScheme()) ) { + String fsPath = FileUtils.getFsPathByMountPath(srcFs.getConf(), srcf.toString()); + String[] path = fsPath.split(Path.SEPARATOR); + srcMountFs = path != null && path.length >1 ? path[1] : null; + } else if("hdfs".equals(srcFs.getUri().getScheme())){ + srcMountFs = srcf.toUri().getAuthority(); + } + if("viewfs".equals(destFs.getUri().getScheme())) { + String fsPath = FileUtils.getFsPathByMountPath(destFs.getConf(), destf.toString()); + String[] path = fsPath.split(Path.SEPARATOR); + destMountFs = path != null && path.length >1 ? path[1] : null; + } else if("hdfs".equals(destFs.getUri().getScheme())) { + destMountFs = destf.toUri().getAuthority(); + } + if(srcMountFs != null && !srcMountFs.equals(destMountFs)) { + return false; + } else { + return true; + } + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index cceea019c2..783f2f4840 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -3596,6 +3596,11 @@ static protected boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, Fil return true; } + //Check if different mount point path in viewfs + if (!FileUtils.equalsMountPointPath(srcf, destf, srcFs, destFs)) { + return true; + } + //Check if different encryption zones HadoopShims.HdfsEncryptionShim srcHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(srcFs); HadoopShims.HdfsEncryptionShim destHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(destFs);