Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision Local version) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision Shelved version) @@ -2196,17 +2196,16 @@ * @throws HiveException If an error occurs while checking for encryption */ private boolean isPathEncrypted(Path path) throws HiveException { - HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; - hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); - if (hdfsEncryptionShim != null) { - try { + try { + HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(path.getFileSystem(conf)); + if (hdfsEncryptionShim != null) { if (hdfsEncryptionShim.isPathEncrypted(path)) { return true; } + } - } catch (Exception e) { + } catch (Exception e) { throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e); - } } return false; Index: shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (revision Local version) +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (revision Shelved version) @@ -512,6 +512,17 @@ public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException; /** + * Checks if two HDFS paths are on the same encrypted or unencrypted zone. + * + * @param path1 Path to HDFS file system + * @param path2 Path to HDFS file system + * @param encryptionShim2 The encryption-shim corresponding to path2. + * @return True if both paths are in the same zone; False otherwise. + * @throws IOException If an error occurred attempting to get encryption information + */ + public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim encryptionShim2) throws IOException; + + /** * Compares two encrypted path strengths. * * @param path1 HDFS path to compare. @@ -562,6 +573,12 @@ @Override public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { /* not supported */ + return true; + } + + @Override + public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim encryptionShim2) throws IOException { + // Not supported. return true; } Index: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (revision Local version) +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (revision Shelved version) @@ -42,6 +42,7 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.locks.ReentrantLock; +import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.ArrayUtils; import org.slf4j.Logger; @@ -211,7 +212,7 @@ /** * Gets information about HDFS encryption */ - private HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; + private Map hdfsEncryptionShims = Maps.newHashMap(); /** * Lineage state. @@ -438,20 +439,31 @@ } public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() throws HiveException { - if (hdfsEncryptionShim == null) { - try { + try { - FileSystem fs = FileSystem.get(sessionConf); + return getHdfsEncryptionShim(FileSystem.get(sessionConf)); + } + catch(HiveException hiveException) { + throw hiveException; + } + catch(Exception exception) { + throw new HiveException(exception); + } + } + + public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim(FileSystem fs) throws HiveException { + if (!hdfsEncryptionShims.containsKey(fs.getUri())) { + try { if ("hdfs".equals(fs.getUri().getScheme())) { - hdfsEncryptionShim = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, sessionConf); + hdfsEncryptionShims.put(fs.getUri(), ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, sessionConf)); } else { - LOG.debug("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem."); + LOG.info("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem."); } } catch (Exception e) { throw new HiveException(e); } } - return hdfsEncryptionShim; + return hdfsEncryptionShims.get(fs.getUri()); } // SessionState is not available in runtime and Hive.get().getConf() is not safe to call Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision Local version) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision Shelved version) @@ -3013,10 +3013,13 @@ } //Check if different encryption zones - HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); + HadoopShims.HdfsEncryptionShim srcHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(srcFs); + HadoopShims.HdfsEncryptionShim destHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(destFs); try { - return hdfsEncryptionShim != null && (hdfsEncryptionShim.isPathEncrypted(srcf) || hdfsEncryptionShim.isPathEncrypted(destf)) - && !hdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf); + return srcHdfsEncryptionShim != null + && destHdfsEncryptionShim != null + && (srcHdfsEncryptionShim.isPathEncrypted(srcf) || destHdfsEncryptionShim.isPathEncrypted(destf)) + && !srcHdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf, destHdfsEncryptionShim); } catch (IOException e) { throw new HiveException(e); } Index: shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (revision Local version) +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (revision Shelved version) @@ -1122,11 +1122,11 @@ @Override public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { - EncryptionZone zone1, zone2; + return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1), + hdfsAdmin.getEncryptionZoneForPath(path2)); + } - zone1 = hdfsAdmin.getEncryptionZoneForPath(path1); - zone2 = hdfsAdmin.getEncryptionZoneForPath(path2); - + private boolean equivalentEncryptionZones(EncryptionZone zone1, EncryptionZone zone2) { if (zone1 == null && zone2 == null) { return true; } else if (zone1 == null || zone2 == null) { @@ -1134,6 +1134,19 @@ } return zone1.equals(zone2); + } + + @Override + public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, + HadoopShims.HdfsEncryptionShim encryptionShim2) throws IOException { + if (!(encryptionShim2 instanceof Hadoop23Shims.HdfsEncryptionShim)) { + LOG.warn("EncryptionShim for path2 (" + path2 + ") is of unexpected type: " + encryptionShim2.getClass() + + ". Assuming path2 is on the same EncryptionZone as path1(" + path1 + ")."); + return true; + } + + return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1), + ((HdfsEncryptionShim)encryptionShim2).hdfsAdmin.getEncryptionZoneForPath(path2)); } @Override