diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 1c91239..655a61e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2262,7 +2262,19 @@ public static boolean renameFile(HiveConf conf, Path srcf, Path destf, FileSyste } } } - success = fs.rename(srcf, destf); + + if ((shims.isPathEncrypted(srcf) || shims.isPathEncrypted(destf)) + && !shims.arePathsOnSameEncryptionZone(srcf, destf)) + { + LOG.info("Copying source " + srcf + " to " + destf + " because HDFS encryption zones are different."); + success = FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf, + true, // delete source + replace, // overwrite destination + conf); + } else { + success = fs.rename(srcf, destf); + } + LOG.info((replace ? "Replacing src:" : "Renaming src:") + srcf.toString() + ";dest: " + destf.toString() + ";Status:" + success); } catch (IOException ioe) { @@ -2362,11 +2374,11 @@ static protected void replaceFiles(Path srcf, Path destf, Path oldPath, HiveConf try { FileSystem fs2 = oldPath.getFileSystem(conf); if (fs2.exists(oldPath)) { - FileUtils.trashFilesUnderDir(fs2, oldPath, conf); + FileUtils.trashFilesUnderDir(fs2, oldPath, conf, Arrays.asList(conf.getVar(HiveConf.ConfVars.STAGINGDIR))); } } catch (Exception e) { //swallow the exception - LOG.warn("Directory " + oldPath.toString() + " canot be removed:" + StringUtils.stringifyException(e)); + LOG.warn("Directory " + oldPath.toString() + " cannot be removed:" + StringUtils.stringifyException(e)); } } @@ -2384,10 +2396,15 @@ static protected void replaceFiles(Path srcf, Path destf, Path oldPath, HiveConf } } - boolean b = renameFile(conf, srcs[0].getPath(), destf, fs, true); - if (!b) { - throw new HiveException("Unable to move results from " + srcs[0].getPath() - + " to destination directory: " + destf); + // Copy/move each file under the source directory to avoid to delete the destination + // directory if it is the root of an HDFS encryption zone. + for (List sdpairs : result) { + for (Path[] sdpair : sdpairs) { + if (!renameFile(conf, sdpair[0], sdpair[1], fs, true)) { + throw new IOException("Unable to move file/directory from " + sdpair[0] + + " to " + sdpair[1]); + } + } } } else { // srcf is a file or pattern containing wildcards if (!fs.exists(destf)) { diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 39953fd..9c21479 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -21,10 +21,7 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.net.InetSocketAddress; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URL; +import java.net.*; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -33,6 +30,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -52,6 +51,8 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobConf; @@ -92,6 +93,16 @@ final boolean zeroCopy; + /** + * Gets information about key encryption metadata + */ + private KeyProvider keyProvider = null; + + /** + * Gets information about HDFS encryption zones + */ + private HdfsAdmin hdfsAdmin = null; + public Hadoop23Shims() { boolean zcr = false; try { @@ -819,6 +830,7 @@ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { ret.put("MAPREDSETUPCLEANUPNEEDED", "mapred.committer.job.setup.cleanup.needed"); } ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed"); + ret.put("HADOOPSECURITYKEYPROVIDER", "hadoop.security.key.provider.path"); return ret; } @@ -849,4 +861,86 @@ public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext con public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException { return FileSystem.newInstance(uri, conf); } + + @Override + public void loadSecurityProviders(FileSystem fs, Configuration conf) throws IOException { + if (keyProvider == null) { + hdfsAdmin = new HdfsAdmin(fs.getUri(), conf); + + String keyProviderPath = conf.get(getHadoopConfNames().get("HADOOPSECURITYKEYPROVIDER"), null); + if (keyProviderPath != null) { + try { + keyProvider = new KMSClientProvider(new URI(keyProviderPath), conf); + } catch (URISyntaxException e) { + throw new IOException("Invalid HDFS security key provider path", e); + } + } + } + } + + @Override + public boolean isPathEncrypted(Path path) throws IOException { + return (hdfsAdmin.getEncryptionZoneForPath(path) != null); + } + + @Override + public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = hdfsAdmin.getEncryptionZoneForPath(path1); + zone2 = hdfsAdmin.getEncryptionZoneForPath(path2); + + if (zone1 == null && zone2 == null) { + return true; + } else if (zone1 == null || zone2 == null) { + return false; + } + + return zone1.equals(zone2); + } + + @Override + public int comparePathKeyStrength(Path path1, Path path2) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = hdfsAdmin.getEncryptionZoneForPath(path1); + zone2 = hdfsAdmin.getEncryptionZoneForPath(path2); + + if (zone1 == null && zone2 == null) { + return 0; + } else if (zone1 == null) { + return -1; + } else if (zone2 == null) { + return 1; + } + + return compareKeyStrength(zone1.getKeyName(), zone2.getKeyName()); + } + + /** + * Compares two encryption key strengths. + * + * @param keyname1 Keyname to compare + * @param keyname2 Keyname to compare + * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. + * @throws IOException If an error occurred attempting to get key metadata + */ + private int compareKeyStrength(String keyname1, String keyname2) throws IOException { + KeyProvider.Metadata meta1, meta2; + + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + + meta1 = keyProvider.getMetadata(keyname1); + meta2 = keyProvider.getMetadata(keyname2); + + if (meta1.getBitLength() < meta2.getBitLength()) { + return -1; + } else if (meta1.getBitLength() == meta2.getBitLength()) { + return 0; + } else { + return 1; + } + } }