From 99c962a278bd09ba3e9007de5c79a61e34791158 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Sun, 19 Apr 2020 00:27:11 -0700 Subject: [PATCH] HIVE-13004 : Remove Encryption Shims --- .../pig/TestHCatLoaderEncryption.java | 7 +- .../hadoop/hive/ql/QTestMiniClusters.java | 18 +- .../org/apache/hadoop/hive/ql/QTestUtil.java | 18 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 19 +- .../metadata/SessionHiveMetaStoreClient.java | 6 +- .../hive/ql/parse/SemanticAnalyzer.java | 32 +-- .../processors/CommandProcessorFactory.java | 11 +- .../hive/ql/processors/CryptoProcessor.java | 51 +++-- .../hadoop/hive/ql/session/SessionState.java | 47 +--- .../ql/util/HiveStrictManagedMigration.java | 23 +- .../hadoop/hive/shims/Hadoop23Shims.java | 205 +----------------- .../org/apache/hadoop/hive/io/HdfsUtils.java | 162 +++++++++++++- .../apache/hadoop/hive/shims/HadoopShims.java | 149 ------------- .../hive/metastore/ReplChangeManager.java | 35 ++- 14 files changed, 260 insertions(+), 523 deletions(-) diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java index beb4fe9f4b..8de60ba394 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java @@ -101,7 +101,7 @@ private static final Map> DISABLED_STORAGE_FORMATS = new HashMap>(); - private String storageFormat; + private final String storageFormat; @Parameterized.Parameters public static Collection generateParameters() { @@ -212,15 +212,10 @@ public void setup() throws Exception { } void initEncryptionShim(HiveConf conf) throws IOException { - FileSystem fs; HadoopShims shims = ShimLoader.getHadoopShims(); conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); int numberOfDataNodes = 4; dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); - fs = dfs.getFileSystem(); - - // set up a java key provider for encrypted hdfs cluster - shims.createHdfsEncryptionShim(fs, conf); } public static String ensurePathEndsInSlash(String path) { diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java index 997b35e18f..23c04f8010 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java @@ -38,9 +38,11 @@ import org.apache.avro.io.EncoderFactory; import org.apache.avro.specific.SpecificDatumWriter; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; import org.apache.hadoop.hive.conf.HiveConf; @@ -95,7 +97,7 @@ private FileSystem fs; private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; - private HadoopShims.HdfsEncryptionShim hes = null; + private KeyProvider keyProvider = null; private MiniLlapCluster llapCluster = null; private MiniDruidCluster druidCluster = null; private SingleNodeKafkaCluster kafkaCluster = null; @@ -113,9 +115,9 @@ TEZ(CoreClusterType.TEZ, FsType.HDFS), TEZ_LOCAL(CoreClusterType.TEZ, FsType.LOCAL), SPARK(CoreClusterType.SPARK, FsType.LOCAL), - MINI_SPARK_ON_YARN(CoreClusterType.SPARK, FsType.HDFS), + MINI_SPARK_ON_YARN(CoreClusterType.SPARK, FsType.HDFS), LLAP(CoreClusterType.TEZ, FsType.HDFS), - LLAP_LOCAL(CoreClusterType.TEZ, FsType.LOCAL), + LLAP_LOCAL(CoreClusterType.TEZ, FsType.LOCAL), NONE(CoreClusterType.MR,FsType.LOCAL), DRUID_LOCAL(CoreClusterType.TEZ, FsType.LOCAL), DRUID(CoreClusterType.TEZ, FsType.HDFS), @@ -427,10 +429,6 @@ public SparkSession getSparkSession() { return sparkSession; } - public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() { - return hes; - } - public HadoopShims.MiniMrShim getMr() { return mr; } @@ -515,7 +513,7 @@ private void setupFileSystem(FsType fsType, HiveConf conf) throws IOException { switch (fsType) { case ENCRYPTED_HDFS: // set up the java key provider for encrypted hdfs cluster - hes = shims.createHdfsEncryptionShim(fs, conf); + keyProvider = ((DistributedFileSystem)fs).getClient().getKeyProvider(); LOG.info("key provider is initialized"); break; case ERASURE_CODED_HDFS: @@ -615,4 +613,8 @@ private void setFsRelatedProperties(HiveConf conf, boolean isLocalFs, FileSystem // TODO Make sure to cleanup created dirs. } + + public KeyProvider getKeyProvider() { + return keyProvider; + } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 38e4bac2cc..c6b842ea3a 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -42,6 +42,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -294,15 +295,14 @@ public void clearPostTestEffects() throws Exception { } public void clearKeysCreatedInTests() { - if (miniClusters.getHdfsEncryptionShim() == null) { - return; - } - try { - for (String keyAlias : miniClusters.getHdfsEncryptionShim().getKeys()) { - miniClusters.getHdfsEncryptionShim().deleteKey(keyAlias); - } - } catch (IOException e) { - LOG.error("Fail to clean the keys created in test due to the error", e); + KeyProvider kp = miniClusters.getKeyProvider(); + if (kp != null) { + try { + for (String key : kp.getKeys()) { + kp.deleteKey(key); + } + } catch (IOException e) { + LOG.error("Fail to clean the keys created in test due to the error", e); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 1f9fb3b897..c72036ec2d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -40,7 +40,6 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.sql.SQLIntegrityConstraintViolationException; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -80,7 +79,6 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileChecksum; @@ -200,7 +198,6 @@ import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; -import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.security.UserGroupInformation; @@ -4062,7 +4059,7 @@ private static void copyFiles(final HiveConf conf, final FileSystem destFs, Arrays.sort(files); for (final FileStatus srcFile : files) { final Path srcP = srcFile.getPath(); - final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs, configuredOwner, isManaged); + final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs, configuredOwner, isManaged, conf); final boolean isRenameAllowed = !needToCopy && !isSrcLocal; @@ -4377,7 +4374,7 @@ public static boolean moveFile(final HiveConf conf, Path srcf, final Path destf, destFs.copyFromLocalFile(srcf, destf); return true; } else { - if (needToCopy(srcf, destf, srcFs, destFs, configuredOwner, isManaged)) { + if (needToCopy(srcf, destf, srcFs, destFs, configuredOwner, isManaged, conf)) { //copy if across file system or encryption zones. LOG.debug("Copying source " + srcf + " to " + destf + " because HDFS encryption zones are different."); return FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf, @@ -4498,12 +4495,12 @@ static private HiveException getHiveException(Exception e, String msg, String lo } /** - * If moving across different FileSystems or differnent encryption zone, need to do a File copy instead of rename. + * If moving across different FileSystems or different encryption zone, need to do a File copy instead of rename. * TODO- consider if need to do this for different file authority. * @throws HiveException */ static private boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, - FileSystem destFs, String configuredOwner, boolean isManaged) throws HiveException { + FileSystem destFs, String configuredOwner, boolean isManaged, HiveConf conf) throws HiveException { //Check if different FileSystems if (!FileUtils.equalsFileSystem(srcFs, destFs)) { return true; @@ -4544,13 +4541,9 @@ static private boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, } //Check if different encryption zones - HadoopShims.HdfsEncryptionShim srcHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(srcFs); - HadoopShims.HdfsEncryptionShim destHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(destFs); try { - return srcHdfsEncryptionShim != null - && destHdfsEncryptionShim != null - && (srcHdfsEncryptionShim.isPathEncrypted(srcf) || destHdfsEncryptionShim.isPathEncrypted(destf)) - && !srcHdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf, destHdfsEncryptionShim); + return (HdfsUtils.isPathEncrypted(srcf, conf) || HdfsUtils.isPathEncrypted(destf, conf)) + && !HdfsUtils.arePathsOnSameEncryptionZone(srcf, destf, conf); } catch (IOException e) { throw new HiveException(e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 3dcf876af3..d058f1eb41 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -79,8 +79,6 @@ import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.shims.HadoopShims; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.ql.stats.StatsUtils; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -730,9 +728,7 @@ private void truncateTempTable(org.apache.hadoop.hive.metastore.api.Table table) Path location = new Path(table.getSd().getLocation()); FileSystem fs = location.getFileSystem(conf); - HadoopShims.HdfsEncryptionShim shim - = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); - if (!shim.isPathEncrypted(location)) { + if (!HdfsUtils.isPathEncrypted(location, conf)) { HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location); FileStatus targetStatus = fs.getFileStatus(location); String targetGroup = targetStatus == null ? null : targetStatus.getGroup(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e13dfe82b7..21761f8a7e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -81,6 +81,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.TransactionalValidationListener; import org.apache.hadoop.hive.metastore.Warehouse; @@ -347,7 +348,7 @@ * that describes percentage and number. */ private final Map nameToSplitSample; - private Map> groupOpToInputTables; + private final Map> groupOpToInputTables; protected Map prunedPartitions; protected List resultSchema; protected CreateViewDesc createVwDesc; @@ -2453,19 +2454,11 @@ private void getMetaData(QB qb, ReadEntity parentInput) * @throws HiveException If an error occurs while checking for encryption */ private boolean isPathEncrypted(Path path) throws HiveException { - try { - HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(path.getFileSystem(conf)); - if (hdfsEncryptionShim != null) { - if (hdfsEncryptionShim.isPathEncrypted(path)) { - return true; - } - } - } catch (Exception e) { - throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e); + return HdfsUtils.isPathEncrypted(path, conf); + } catch (IOException e) { + throw new HiveException("Unable to find if path is encrypted : " + path, e); } - - return false; } /** @@ -2477,18 +2470,11 @@ private boolean isPathEncrypted(Path path) throws HiveException { * @throws HiveException If an error occurs while comparing key strengths. */ private int comparePathKeyStrength(Path p1, Path p2) throws HiveException { - HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; - - hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); - if (hdfsEncryptionShim != null) { - try { - return hdfsEncryptionShim.comparePathKeyStrength(p1, p2); - } catch (Exception e) { - throw new HiveException("Unable to compare key strength for " + p1 + " and " + p2 + " : " + e, e); - } + try { + return HdfsUtils.comparePathKeyStrength(p1, p2, conf); + } catch (IOException e) { + throw new HiveException("Unable to compare key strength for " + p1 + " and " + p2 + " : " + e, e); } - - return 0; // Non-encrypted path (or equals strength) } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java index 977ab5372d..2ad675197a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java @@ -26,12 +26,9 @@ import java.util.Set; import javax.annotation.Nonnull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.DriverFactory; -import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.session.SessionState; /** @@ -95,11 +92,7 @@ public static CommandProcessor getForHiveCommandInternal(String[] cmd, HiveConf case RELOAD: return new ReloadProcessor(); case CRYPTO: - try { - return new CryptoProcessor(SessionState.get().getHdfsEncryptionShim(), conf); - } catch (HiveException e) { - throw new SQLException("Fail to start the command processor due to the exception: ", e); - } + return new CryptoProcessor(conf); case ERASURE: try { return new ErasureProcessor(conf); @@ -111,8 +104,6 @@ public static CommandProcessor getForHiveCommandInternal(String[] cmd, HiveConf } } - private static Logger LOG = LoggerFactory.getLogger(CommandProcessorFactory.class); - public static CommandProcessor get(String[] cmd, @Nonnull HiveConf conf) throws SQLException { CommandProcessor result = getForHiveCommand(cmd, conf); if (result != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java index 59e2559f45..8a3974c119 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java @@ -27,7 +27,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.HadoopShims; @@ -41,27 +43,40 @@ public class CryptoProcessor implements CommandProcessor { public static final Logger LOG = LoggerFactory.getLogger(CryptoProcessor.class.getName()); - private HadoopShims.HdfsEncryptionShim encryptionShim; + private final HiveConf conf; + private final Options CREATE_KEY_OPTIONS; + private final Options DELETE_KEY_OPTIONS; + private final Options CREATE_ZONE_OPTIONS; - private Options CREATE_KEY_OPTIONS; - private Options DELETE_KEY_OPTIONS; - private Options CREATE_ZONE_OPTIONS; + private final int DEFAULT_BIT_LENGTH = 128; - private int DEFAULT_BIT_LENGTH = 128; - - public CryptoProcessor(HadoopShims.HdfsEncryptionShim encryptionShim, HiveConf conf) { - this.encryptionShim = encryptionShim; + public CryptoProcessor(HiveConf config) { + this.conf = config; CREATE_KEY_OPTIONS = new Options(); - CREATE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); - CREATE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("bitLength").create()); // optional + OptionBuilder.hasArg(); + OptionBuilder.withLongOpt("keyName"); + OptionBuilder.isRequired(); + CREATE_KEY_OPTIONS.addOption(OptionBuilder.create()); + OptionBuilder.hasArg(); + OptionBuilder.withLongOpt("bitLength"); + CREATE_KEY_OPTIONS.addOption(OptionBuilder.create()); // optional DELETE_KEY_OPTIONS = new Options(); - DELETE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); + OptionBuilder.hasArg(); + OptionBuilder.withLongOpt("keyName"); + OptionBuilder.isRequired(); + DELETE_KEY_OPTIONS.addOption(OptionBuilder.create()); CREATE_ZONE_OPTIONS = new Options(); - CREATE_ZONE_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); - CREATE_ZONE_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("path").isRequired().create()); + OptionBuilder.hasArg(); + OptionBuilder.withLongOpt("keyName"); + OptionBuilder.isRequired(); + CREATE_ZONE_OPTIONS.addOption(OptionBuilder.create()); + OptionBuilder.hasArg(); + OptionBuilder.withLongOpt("path"); + OptionBuilder.isRequired(); + CREATE_ZONE_OPTIONS.addOption(OptionBuilder.create()); } private CommandLine parseCommandArgs(final Options opts, String[] args) throws ParseException { @@ -81,9 +96,9 @@ public CommandProcessorResponse run(String command) throws CommandProcessorExcep throw new CommandProcessorException("Encryption Processor Helper Failed: Command arguments are empty."); } - if (encryptionShim == null) { + if (DFSUtilClient.isHDFSEncryptionEnabled(conf)) { throw new CommandProcessorException( - "Encryption Processor Helper Failed: Hadoop encryption shim is not initialized."); + "Encryption Processor Helper Failed: Encryption is not enabled."); } String action = args[0]; @@ -119,7 +134,7 @@ private void createEncryptionKey(String[] params) throws Exception { String bitLength = args.getOptionValue("bitLength", Integer.toString(DEFAULT_BIT_LENGTH)); try { - encryptionShim.createKey(keyName, Integer.parseInt(bitLength)); + HdfsUtils.createKey(keyName, Integer.parseInt(bitLength), conf); } catch (Exception e) { throw new Exception("Cannot create encryption key: " + e.getMessage()); } @@ -144,7 +159,7 @@ private void createEncryptionZone(String[] params) throws Exception { Path cryptoZone = new Path(cryptoZoneStr); try { - encryptionShim.createEncryptionZone(cryptoZone, keyName); + HdfsUtils.createEncryptionZone(cryptoZone, keyName, conf); } catch (IOException e) { throw new Exception("Cannot create encryption zone: " + e.getMessage()); } @@ -163,7 +178,7 @@ private void deleteEncryptionKey(String[] params) throws Exception { String keyName = args.getOptionValue("keyName"); try { - encryptionShim.deleteKey(keyName); + HdfsUtils.deleteKey(keyName, conf); } catch (IOException e) { throw new Exception("Cannot delete encryption key: " + e.getMessage()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 6c6138ef56..af3d82c1a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -49,6 +49,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -102,16 +104,12 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext.CLIENT_TYPE; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl; import org.apache.hadoop.hive.ql.util.ResourceDownloader; -import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; /** * SessionState encapsulates common data associated with a session. @@ -172,7 +170,7 @@ /** * The flag to indicate if the session already started so we can skip the init */ - private AtomicBoolean isStarted = new AtomicBoolean(false); + private final AtomicBoolean isStarted = new AtomicBoolean(false); /* * HiveHistory Object */ @@ -250,16 +248,6 @@ private SparkSession sparkSession; - /** - * Gets information about HDFS encryption - */ - private Map hdfsEncryptionShims = Maps.newHashMap(); - - /** - * Cache for Erasure Coding shims. - */ - private Map erasureCodingShims; - private final String userName; /** @@ -321,7 +309,7 @@ private String atsDomainId; - private List cleanupItems = new LinkedList(); + private final List cleanupItems = new LinkedList(); private final AtomicLong sparkSessionId = new AtomicLong(); @@ -522,33 +510,6 @@ public HiveTxnManager setTxnMgr(HiveTxnManager mgr) { txnMgr = mgr; return tmp; } - public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() throws HiveException { - try { - return getHdfsEncryptionShim(FileSystem.get(sessionConf)); - } - catch(HiveException hiveException) { - throw hiveException; - } - catch(Exception exception) { - throw new HiveException(exception); - } - } - - public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim(FileSystem fs) throws HiveException { - if (!hdfsEncryptionShims.containsKey(fs.getUri())) { - try { - if ("hdfs".equals(fs.getUri().getScheme())) { - hdfsEncryptionShims.put(fs.getUri(), ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, sessionConf)); - } else { - LOG.info("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem."); - } - } catch (Exception e) { - throw new HiveException(e); - } - } - - return hdfsEncryptionShims.get(fs.getUri()); - } // SessionState is not available in runtime and Hive.get().getConf() is not safe to call private static class SessionStates { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java index 2c1ba5fc48..d34beecb48 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.cli.CommonCliOptions; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; @@ -179,19 +180,16 @@ public String toString() { final boolean shouldModifyManagedTableLocation; final boolean shouldMoveExternal; final Path targetPath; - final HadoopShims.HdfsEncryptionShim encryptionShim; final HadoopShims.HdfsErasureCodingShim ecShim; WarehouseRootCheckResult( boolean shouldModifyManagedTableLocation, boolean shouldMoveExternal, Path curWhRootPath, - HadoopShims.HdfsEncryptionShim encryptionShim, HadoopShims.HdfsErasureCodingShim ecShim) { this.shouldModifyManagedTableLocation = shouldModifyManagedTableLocation; this.shouldMoveExternal = shouldMoveExternal; this.targetPath = curWhRootPath; - this.encryptionShim = encryptionShim; this.ecShim = ecShim; } } @@ -425,7 +423,6 @@ private static int getIntOptionValue(CommandLine commandLine, String optionName, private final RunOptions runOptions; private final boolean createExternalDirsForDbs; private final Path targetPath; - private final HadoopShims.HdfsEncryptionShim encryptionShim; private final HadoopShims.HdfsErasureCodingShim ecShim; private final String ownerName; private final String groupName; @@ -450,7 +447,6 @@ private static int getIntOptionValue(CommandLine commandLine, String optionName, this.dirPerms = ownerPermsOptions.dirPerms; this.filePerms = ownerPermsOptions.filePerms; this.targetPath = warehouseRootCheckResult.targetPath; - this.encryptionShim = warehouseRootCheckResult.encryptionShim; this.ecShim = warehouseRootCheckResult.ecShim; // Make sure all --hiveconf settings get added to the HiveConf. @@ -534,7 +530,6 @@ static WarehouseRootCheckResult checkOldWarehouseRoot(RunOptions runOptions, Hiv boolean shouldModifyManagedTableLocation = runOptions.shouldModifyManagedTableLocation; boolean shouldMoveExternal = runOptions.shouldMoveExternal; Path targetPath = null; - HadoopShims.HdfsEncryptionShim encryptionShim = null; HadoopShims.HdfsErasureCodingShim ecShim = null; if (shouldMoveExternal && !checkExternalWarehouseDir(conf)) { @@ -578,8 +573,7 @@ static WarehouseRootCheckResult checkOldWarehouseRoot(RunOptions runOptions, Hiv shouldModifyManagedTableLocation = false; shouldMoveExternal = false; } else { - encryptionShim = ShimLoader.getHadoopShims().createHdfsEncryptionShim(oldWhRootFs, conf); - if (!hasEquivalentEncryption(encryptionShim, oldWhRootPath, targetPath)) { + if (!hasEquivalentEncryption(oldWhRootPath, targetPath, conf)) { LOG.info("oldWarehouseRoot {} and target path {} have different encryption zones." + " Disabling shouldModifyManagedTableLocation and shouldMoveExternal", oldWhRootPath, targetPath); @@ -602,7 +596,7 @@ static WarehouseRootCheckResult checkOldWarehouseRoot(RunOptions runOptions, Hiv } return new WarehouseRootCheckResult(shouldModifyManagedTableLocation, shouldMoveExternal, - targetPath, encryptionShim, ecShim); + targetPath, ecShim); } static OwnerPermsOptions checkOwnerPermsOptions(RunOptions runOptions, HiveConf conf) { @@ -799,7 +793,7 @@ boolean shouldModifyDatabaseLocation(Database dbObj) throws IOException, MetaExc String dbLocation = dbObj.getLocationUri(); Path oldDefaultDbLocation = oldWh.get().getDefaultDatabasePath(dbName); if (arePathsEqual(conf, dbLocation, oldDefaultDbLocation.toString())) { - if (hasEquivalentEncryption(encryptionShim, oldDefaultDbLocation, targetPath)) { + if (hasEquivalentEncryption( oldDefaultDbLocation, targetPath, conf)) { if (hasEquivalentErasureCodingPolicy(ecShim, oldDefaultDbLocation, targetPath)) { return true; } else { @@ -823,7 +817,7 @@ boolean shouldModifyTableLocation(Database dbObj, Table tableObj) throws IOExcep String tableLocation = tableObj.getSd().getLocation(); Path oldDefaultTableLocation = oldWh.get().getDefaultTablePath(dbObj, tableObj.getTableName()); if (arePathsEqual(conf, tableLocation, oldDefaultTableLocation.toString())) { - if (hasEquivalentEncryption(encryptionShim, oldDefaultTableLocation, targetPath)) { + if (hasEquivalentEncryption(oldDefaultTableLocation, targetPath, conf)) { if (hasEquivalentErasureCodingPolicy(ecShim, oldDefaultTableLocation, targetPath)) { return true; } else { @@ -1616,11 +1610,10 @@ static FileStatus getFileStatus(FileSystem fs, Path path) throws IOException { return fs.listStatus(path); } - static boolean hasEquivalentEncryption(HadoopShims.HdfsEncryptionShim encryptionShim, - Path path1, Path path2) throws IOException { + static boolean hasEquivalentEncryption(Path path1, Path path2, HiveConf conf) throws IOException { // Assumes these are both qualified paths are in the same FileSystem - if (encryptionShim.isPathEncrypted(path1) || encryptionShim.isPathEncrypted(path2)) { - if (!encryptionShim.arePathsOnSameEncryptionZone(path1, path2)) { + if (HdfsUtils.isPathEncrypted(path1, conf) || HdfsUtils.isPathEncrypted(path2, conf)) { + if (!HdfsUtils.arePathsOnSameEncryptionZone(path1, path2, conf)) { return false; } } diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 2eafef0fc1..e928af707c 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -27,7 +27,6 @@ import java.net.URI; import java.nio.ByteBuffer; import java.security.AccessControlException; -import java.security.NoSuchAlgorithmException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Comparator; @@ -41,9 +40,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.key.KeyProvider; -import org.apache.hadoop.crypto.key.KeyProvider.Options; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.DefaultFileAccess; @@ -65,7 +62,6 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -265,11 +261,6 @@ public void refreshDefaultQueue(Configuration conf, String userName) throws IOEx //no op } - private boolean isFairScheduler (Configuration conf) { - return "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler". - equalsIgnoreCase(conf.get(YarnConfiguration.RM_SCHEDULER)); - } - /** * Returns a shim to wrap MiniMrCluster */ @@ -1204,195 +1195,6 @@ public static boolean isHdfsEncryptionSupported() { return hdfsEncryptionSupport; } - public class HdfsEncryptionShim implements HadoopShims.HdfsEncryptionShim { - private final String HDFS_SECURITY_DEFAULT_CIPHER = "AES/CTR/NoPadding"; - - /** - * Gets information about HDFS encryption zones - */ - private HdfsAdmin hdfsAdmin = null; - - /** - * Used to compare encryption key strengths. - */ - private KeyProvider keyProvider = null; - - private final Configuration conf; - - public HdfsEncryptionShim(URI uri, Configuration conf) throws IOException { - DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.get(uri, conf); - - this.conf = conf; - this.keyProvider = dfs.getClient().getKeyProvider(); - this.hdfsAdmin = new HdfsAdmin(uri, conf); - } - - @Override - public boolean isPathEncrypted(Path path) throws IOException { - Path fullPath; - if (path.isAbsolute()) { - fullPath = path; - } else { - fullPath = path.getFileSystem(conf).makeQualified(path); - } - if(!"hdfs".equalsIgnoreCase(path.toUri().getScheme())) { - return false; - } - - return (getEncryptionZoneForPath(fullPath) != null); - } - - public EncryptionZone getEncryptionZoneForPath(Path path) throws IOException { - if (path.getFileSystem(conf).exists(path)) { - return hdfsAdmin.getEncryptionZoneForPath(path); - } else if (!path.getParent().equals(path)) { - return getEncryptionZoneForPath(path.getParent()); - } else { - return null; - } - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { - return equivalentEncryptionZones(getEncryptionZoneForPath(path1), - getEncryptionZoneForPath(path2)); - } - - private boolean equivalentEncryptionZones(EncryptionZone zone1, EncryptionZone zone2) { - if (zone1 == null && zone2 == null) { - return true; - } else if (zone1 == null || zone2 == null) { - return false; - } - - return zone1.equals(zone2); - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, - HadoopShims.HdfsEncryptionShim encryptionShim2) throws IOException { - if (!(encryptionShim2 instanceof Hadoop23Shims.HdfsEncryptionShim)) { - LOG.warn("EncryptionShim for path2 (" + path2 + ") is of unexpected type: " + encryptionShim2.getClass() - + ". Assuming path2 is on the same EncryptionZone as path1(" + path1 + ")."); - return true; - } - - return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1), - ((HdfsEncryptionShim)encryptionShim2).hdfsAdmin.getEncryptionZoneForPath(path2)); - } - - /** - * Compares two encryption key strengths. - * - * @param path1 First path to compare - * @param path2 Second path to compare - * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. - * @throws IOException If an error occurred attempting to get key metadata - */ - @Override - public int comparePathKeyStrength(Path path1, Path path2) throws IOException { - EncryptionZone zone1, zone2; - - zone1 = getEncryptionZoneForPath(path1); - zone2 = getEncryptionZoneForPath(path2); - - if (zone1 == null && zone2 == null) { - return 0; - } else if (zone1 == null) { - return -1; - } else if (zone2 == null) { - return 1; - } - - return compareKeyStrength(zone1, zone2); - } - - @Override - public void createEncryptionZone(Path path, String keyName) throws IOException { - hdfsAdmin.createEncryptionZone(path, keyName); - } - - @Override - public void createKey(String keyName, int bitLength) - throws IOException, NoSuchAlgorithmException { - - checkKeyProvider(); - - if (keyProvider.getMetadata(keyName) == null) { - final KeyProvider.Options options = new Options(this.conf); - options.setCipher(HDFS_SECURITY_DEFAULT_CIPHER); - options.setBitLength(bitLength); - keyProvider.createKey(keyName, options); - keyProvider.flush(); - } else { - throw new IOException("key '" + keyName + "' already exists"); - } - } - - @Override - public void deleteKey(String keyName) throws IOException { - checkKeyProvider(); - - if (keyProvider.getMetadata(keyName) != null) { - keyProvider.deleteKey(keyName); - keyProvider.flush(); - } else { - throw new IOException("key '" + keyName + "' does not exist."); - } - } - - @Override - public List getKeys() throws IOException { - checkKeyProvider(); - return keyProvider.getKeys(); - } - - private void checkKeyProvider() throws IOException { - if (keyProvider == null) { - throw new IOException("HDFS security key provider is not configured on your server."); - } - } - - /** - * Compares two encryption key strengths. - * - * @param zone1 First EncryptionZone to compare - * @param zone2 Second EncryptionZone to compare - * @return 1 if zone1 is stronger; 0 if zones are equal; -1 if zone1 is weaker. - * @throws IOException If an error occurred attempting to get key metadata - */ - private int compareKeyStrength(EncryptionZone zone1, EncryptionZone zone2) throws IOException { - - // zone1, zone2 should already have been checked for nulls. - assert zone1 != null && zone2 != null : "Neither EncryptionZone under comparison can be null."; - - CipherSuite suite1 = zone1.getSuite(); - CipherSuite suite2 = zone2.getSuite(); - - if (suite1 == null && suite2 == null) { - return 0; - } else if (suite1 == null) { - return -1; - } else if (suite2 == null) { - return 1; - } - - return Integer.compare(suite1.getAlgorithmBlockSize(), suite2.getAlgorithmBlockSize()); - } - } - - @Override - public HadoopShims.HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException { - if (isHdfsEncryptionSupported()) { - URI uri = fs.getUri(); - if ("hdfs".equals(uri.getScheme())) { - return new HdfsEncryptionShim(uri, conf); - } - } - - return new HadoopShims.NoopHdfsEncryptionShim(); - } - @Override public Path getPathWithoutSchemeAndAuthority(Path path) { return Path.getPathWithoutSchemeAndAuthority(path); @@ -1453,7 +1255,7 @@ public UserGroupInformation cloneUgi(UserGroupInformation baseUgi) throws IOExce } try { Subject origSubject = (Subject) getSubjectMethod.invoke(baseUgi); - + Subject subject = new Subject(false, origSubject.getPrincipals(), cloneCredentials(origSubject.getPublicCredentials()), cloneCredentials(origSubject.getPrivateCredentials())); @@ -1471,7 +1273,7 @@ public UserGroupInformation cloneUgi(UserGroupInformation baseUgi) throws IOExce } return set; } - + private static Boolean hdfsErasureCodingSupport; /** @@ -1551,10 +1353,7 @@ public String getStatus() { */ private HdfsAdmin hdfsAdmin = null; - private final Configuration conf; - HdfsErasureCodingShim(URI uri, Configuration conf) throws IOException { - this.conf = conf; this.hdfsAdmin = new HdfsAdmin(uri, conf); } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java index e59eb327f8..cc040627b0 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.io; import java.io.IOException; +import java.net.URI; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -28,6 +30,8 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProvider.Options; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; @@ -38,7 +42,9 @@ import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; - +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -224,4 +230,158 @@ AclStatus getAclStatus() { return this.aclStatus; } } + + /** + * Compares two encryption key strengths. + * + * @param keyname1 Keyname to compare + * @param keyname2 Keyname to compare + * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. + * @throws IOException If an error occurred attempting to get key metadata + */ + public static int comparePathKeyStrength(Path path1, Path path2, Configuration conf) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = getEZ(path1, conf); + zone2 = getEZ(path2, conf); + + if (zone1 == null && zone2 == null) { + return 0; + } else if (zone1 == null) { + return -1; + } else if (zone2 == null) { + return 1; + } + + KeyProvider keyProvider = ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider(); + KeyProvider.Metadata meta1, meta2; + + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + + meta1 = keyProvider.getMetadata(zone1.getKeyName()); + meta2 = keyProvider.getMetadata(zone2.getKeyName()); + + if (meta1.getBitLength() < meta2.getBitLength()) { + return -1; + } else if (meta1.getBitLength() == meta2.getBitLength()) { + return 0; + } else { + return 1; + } + } + + + /** + * create encryption zone by path and keyname + * @param path HDFS path to create encryption zone + * @param keyName keyname + * @throws IOException + */ + public static void createEncryptionZone(Path path, String keyName, Configuration conf) throws IOException { + getHDFSAdmin(path, conf).createEncryptionZone(path, keyName); + } + + /** + * Checks if two HDFS paths are on the same encrypted or unencrypted zone. + * + * @param path1 Path to HDFS file system + * @param path2 Path to HDFS file system + * @return True if both paths are in the same zone; False otherwise. + * @throws IOException If an error occurred attempting to get encryption information + */ + public static boolean arePathsOnSameEncryptionZone(Path path1, Path path2, Configuration conf) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = getHDFSAdmin(path1, conf).getEncryptionZoneForPath(path1); + zone2 = getHDFSAdmin(path2, conf).getEncryptionZoneForPath(path2); + + if (zone1 == null && zone2 == null) { + return true; + } else if (zone1 == null || zone2 == null) { + return false; + } + + return zone1.equals(zone2); + } + private final static String HDFS_SECURITY_DEFAULT_CIPHER = "AES/CTR/NoPadding"; + + /** + * Creates an encryption key. + * + * @param keyName Name of the key + * @param bitLength Key encryption length in bits (128 or 256). + * @throws IOException If an error occurs while creating the encryption key + * @throws NoSuchAlgorithmException If cipher algorithm is invalid. + */ + public static void createKey(String keyName, int bitLength, Configuration conf) + throws IOException, NoSuchAlgorithmException { + + KeyProvider keyProvider = ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider(); + + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + if (keyProvider.getMetadata(keyName) == null) { + final KeyProvider.Options options = new Options(conf); + options.setCipher(HDFS_SECURITY_DEFAULT_CIPHER); + options.setBitLength(bitLength); + keyProvider.createKey(keyName, options); + keyProvider.flush(); + } else { + throw new IOException("key '" + keyName + "' already exists"); + } + } + + public static void deleteKey(String keyName, Configuration conf) throws IOException { + + KeyProvider keyProvider = ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider(); + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + if (keyProvider.getMetadata(keyName) != null) { + keyProvider.deleteKey(keyName); + keyProvider.flush(); + } else { + throw new IOException("key '" + keyName + "' does not exist."); + } + } + + public static EncryptionZone getEZ(Path path, Configuration conf) throws IOException { + + HdfsAdmin admin = getHDFSAdmin(path, conf); + if (null == admin) { + return null; + } + + Path fullPath; + if (path.isAbsolute()) { + fullPath = path; + } else { + fullPath = path.getFileSystem(conf).makeQualified(path); + } + return admin.getEncryptionZoneForPath(fullPath); + } + + /** + * Checks if a given HDFS path is encrypted. + * + * @param path Path to HDFS file system + * @return True if it is encrypted; False otherwise. + * @throws IOException If an error occurred attempting to get encryption information + */ + + public static boolean isPathEncrypted(Path path, Configuration conf) throws IOException { + + return getEZ(path, conf) != null; + } + private static HdfsAdmin getHDFSAdmin (Path path, Configuration conf) throws IOException { + + URI uri = path.toUri(); + if(!"hdfs".equalsIgnoreCase(uri.getScheme())) { + return null; + } + return new HdfsAdmin(uri, conf); + } } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 87842138c3..af1c70c668 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -23,15 +23,12 @@ import java.net.URI; import java.nio.ByteBuffer; import java.security.AccessControlException; -import java.security.NoSuchAlgorithmException; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.TreeMap; -import com.google.common.annotations.VisibleForTesting; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; @@ -42,7 +39,6 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobConf; @@ -532,151 +528,6 @@ boolean runDistCpAs(List srcPaths, Path dst, Configuration conf, UserGroup */ public boolean runDistCp(List srcPaths, Path dst, Configuration conf) throws IOException; - /** - * This interface encapsulates methods used to get encryption information from - * HDFS paths. - */ - public interface HdfsEncryptionShim { - /** - * Checks if a given HDFS path is encrypted. - * - * @param path Path to HDFS file system - * @return True if it is encrypted; False otherwise. - * @throws IOException If an error occurred attempting to get encryption information - */ - public boolean isPathEncrypted(Path path) throws IOException; - - /** - * Checks if two HDFS paths are on the same encrypted or unencrypted zone. - * - * @param path1 Path to HDFS file system - * @param path2 Path to HDFS file system - * @return True if both paths are in the same zone; False otherwise. - * @throws IOException If an error occurred attempting to get encryption information - */ - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException; - - /** - * Checks if two HDFS paths are on the same encrypted or unencrypted zone. - * - * @param path1 Path to HDFS file system - * @param path2 Path to HDFS file system - * @param encryptionShim2 The encryption-shim corresponding to path2. - * @return True if both paths are in the same zone; False otherwise. - * @throws IOException If an error occurred attempting to get encryption information - */ - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim encryptionShim2) throws IOException; - - /** - * Compares two encrypted path strengths. - * - * @param path1 HDFS path to compare. - * @param path2 HDFS path to compare. - * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. - * @throws IOException If an error occurred attempting to get encryption/key metadata - */ - public int comparePathKeyStrength(Path path1, Path path2) throws IOException; - - /** - * Create encryption zone by path and keyname. - * @param path HDFS path to create encryption zone - * @param keyName keyname - * @throws IOException - */ - @VisibleForTesting - public void createEncryptionZone(Path path, String keyName) throws IOException; - - /** - * Get encryption zone by path. - * @param path HDFS path to create encryption zone. - * @throws IOException - */ - EncryptionZone getEncryptionZoneForPath(Path path) throws IOException; - - /** - * Creates an encryption key. - * - * @param keyName Name of the key - * @param bitLength Key encryption length in bits (128 or 256). - * @throws IOException If an error occurs while creating the encryption key - * @throws NoSuchAlgorithmException If cipher algorithm is invalid. - */ - @VisibleForTesting - public void createKey(String keyName, int bitLength) - throws IOException, NoSuchAlgorithmException; - - @VisibleForTesting - public void deleteKey(String keyName) throws IOException; - - @VisibleForTesting - public List getKeys() throws IOException; - } - - /** - * This is a dummy class used when the hadoop version does not support hdfs encryption. - */ - public static class NoopHdfsEncryptionShim implements HdfsEncryptionShim { - @Override - public boolean isPathEncrypted(Path path) throws IOException { - /* not supported */ - return false; - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { - /* not supported */ - return true; - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim encryptionShim2) throws IOException { - // Not supported. - return true; - } - - @Override - public int comparePathKeyStrength(Path path1, Path path2) throws IOException { - /* not supported */ - return 0; - } - - @Override - public void createEncryptionZone(Path path, String keyName) { - /* not supported */ - } - - @Override - public EncryptionZone getEncryptionZoneForPath(Path path) throws IOException { - return null; - } - - @Override - public void createKey(String keyName, int bitLength) { - /* not supported */ - } - - @Override - public void deleteKey(String keyName) throws IOException { - /* not supported */ - } - - @Override - public List getKeys() throws IOException{ - /* not supported */ - return null; - } - } - - /** - * Returns a new instance of the HdfsEncryption shim. - * - * @param fs A FileSystem object to HDFS - * @param conf A Configuration object - * @return A new instance of the HdfsEncryption shim. - * @throws IOException If an error occurred while creating the instance. - */ - public HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException; - /** * Information about an Erasure Coding Policy. */ diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java index 8e1bb4e493..4a1bef04e1 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; @@ -45,10 +46,7 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.Retry; import org.apache.hadoop.hive.metastore.utils.StringUtils; -import org.apache.hadoop.hive.shims.HadoopShims; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.hive.shims.HadoopShims.HdfsEncryptionShim; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +58,6 @@ private static boolean inited = false; private static boolean enabled = false; private static Map encryptionZoneToCmrootMapping = new HashMap<>(); - private static HadoopShims hadoopShims = ShimLoader.getHadoopShims(); private static Configuration conf; private String msUser; private String msGroup; @@ -82,12 +79,12 @@ } public static class FileInfo { - private FileSystem srcFs; - private Path sourcePath; - private Path cmPath; - private String checkSum; + private final FileSystem srcFs; + private final Path sourcePath; + private final Path cmPath; + private final String checkSum; private boolean useSourcePath; - private String subDir; + private final String subDir; private boolean copyDone; public FileInfo(FileSystem srcFs, Path sourcePath, String subDir) { @@ -165,12 +162,10 @@ private ReplChangeManager(Configuration conf) throws MetaException { Path cmroot = new Path(cmRootDir); createCmRoot(cmroot); FileSystem cmRootFs = cmroot.getFileSystem(conf); - HdfsEncryptionShim pathEncryptionShim = hadoopShims - .createHdfsEncryptionShim(cmRootFs, conf); - if (pathEncryptionShim.isPathEncrypted(cmroot)) { + if (HdfsUtils.isPathEncrypted(cmroot, conf)) { //If cm root is encrypted we keep using it for the encryption zone String encryptionZonePath = cmRootFs.getUri() - + pathEncryptionShim.getEncryptionZoneForPath(cmroot).getPath(); + + HdfsUtils.getEZ(cmroot, conf).getPath(); encryptionZoneToCmrootMapping.put(encryptionZonePath, cmRootDir); } else { encryptionZoneToCmrootMapping.put(NO_ENCRYPTION, cmRootDir); @@ -181,7 +176,7 @@ private ReplChangeManager(Configuration conf) throws MetaException { throw new MetaException(ConfVars.REPLCMENCRYPTEDDIR.getHiveName() + " should be absolute path"); } createCmRoot(cmRootFallback); - if (pathEncryptionShim.isPathEncrypted(cmRootFallback)) { + if (HdfsUtils.isPathEncrypted(cmRootFallback, conf)) { throw new MetaException(ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR.getHiveName() + " should not be encrypted"); } @@ -203,6 +198,7 @@ private ReplChangeManager(Configuration conf) throws MetaException { // 1. For har files, _index and _masterindex is required files // 2. _success file is required for Oozie to indicate availability of data source private static final PathFilter hiddenFileFilter = new PathFilter(){ + @Override public boolean accept(Path p){ return !p.getName().startsWith("."); } @@ -453,9 +449,9 @@ public static boolean isCMFileUri(Path fromPath) { * Thread to clear old files of cmroot recursively */ static class CMClearer implements Runnable { - private Map encryptionZones; - private long secRetain; - private Configuration conf; + private final Map encryptionZones; + private final long secRetain; + private final Configuration conf; CMClearer(Map encryptionZones, long secRetain, Configuration conf) { this.encryptionZones = encryptionZones; @@ -557,10 +553,9 @@ static Path getCmRoot(Path path) throws IOException { String cmrootDir = fallbackNonEncryptedCmRootDir; String encryptionZonePath = NO_ENCRYPTION; if (enabled) { - HdfsEncryptionShim pathEncryptionShim = hadoopShims.createHdfsEncryptionShim(path.getFileSystem(conf), conf); - if (pathEncryptionShim.isPathEncrypted(path)) { + if (HdfsUtils.isPathEncrypted(path, conf)) { encryptionZonePath = path.getFileSystem(conf).getUri() - + pathEncryptionShim.getEncryptionZoneForPath(path).getPath(); + + HdfsUtils.getEZ(path, conf).getPath(); //For encryption zone, create cm at the relative path specified by hive.repl.cm.encryptionzone.rootdir //at the root of the encryption zone cmrootDir = encryptionZonePath + Path.SEPARATOR + encryptedCmRootDir; -- 2.17.2 (Apple Git-113)