From 2426242a57f8e5cf4adf5f295198dd937e6cb4f3 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Wed, 10 Feb 2016 21:57:00 -0800 Subject: [PATCH] HIVE-13004 : Remove Encryption shims --- .../hcatalog/pig/TestHCatLoaderEncryption.java | 10 +- .../java/org/apache/hadoop/hive/ql/QTestUtil.java | 27 ++- .../hadoop/hive/metastore/HiveMetaStore.java | 6 +- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 29 ++- .../ql/metadata/SessionHiveMetaStoreClient.java | 12 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 28 +-- .../ql/processors/CommandProcessorFactory.java | 6 +- .../hadoop/hive/ql/processors/CryptoProcessor.java | 32 +--- .../hadoop/hive/ql/session/SessionState.java | 43 +---- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 204 --------------------- .../java/org/apache/hadoop/hive/io/HdfsUtils.java | 172 ++++++++++++++++- .../org/apache/hadoop/hive/shims/HadoopShims.java | 136 -------------- 12 files changed, 219 insertions(+), 486 deletions(-) diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java index 496f3c85da..07f514a8df 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java @@ -91,7 +91,6 @@ private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri"; private HadoopShims.MiniDFSShim dfs = null; - private HadoopShims.HdfsEncryptionShim hes = null; private final String[] testOnlyCommands = new String[]{"crypto"}; private Driver driver; private Map> basicInputData; @@ -100,7 +99,7 @@ private static final Map> DISABLED_STORAGE_FORMATS = new HashMap>(); - private String storageFormat; + private final String storageFormat; @Parameterized.Parameters public static Collection generateParameters() { @@ -172,7 +171,7 @@ public void setup() throws Exception { String s = hiveConf.get("hdfs.minidfs.basedir"); if(s == null || s.length() <= 0) { //return System.getProperty("test.build.data", "build/test/data") + "/dfs/"; - hiveConf.set("hdfs.minidfs.basedir", + hiveConf.set("hdfs.minidfs.basedir", System.getProperty("test.build.data", "build/test/data") + "_" + System.currentTimeMillis() + "_" + salt.getAndIncrement() + "/dfs/"); } @@ -214,15 +213,10 @@ public void setup() throws Exception { } void initEncryptionShim(HiveConf conf) throws IOException { - FileSystem fs; HadoopShims shims = ShimLoader.getHadoopShims(); conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); int numberOfDataNodes = 4; dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null); - fs = dfs.getFileSystem(); - - // set up a java key provider for encrypted hdfs cluster - hes = shims.createHdfsEncryptionShim(fs, conf); } public static String ensurePathEndsInSlash(String path) { diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 44779548ff..31868c9763 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -38,6 +38,7 @@ import java.io.PrintStream; import java.io.Serializable; import java.io.StringWriter; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystems; @@ -89,6 +90,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hive.cli.CliDriver; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; @@ -100,6 +102,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Index; @@ -190,7 +193,6 @@ private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; private FileSystem fs; - private HadoopShims.HdfsEncryptionShim hes = null; private MiniLlapCluster llapCluster = null; private String hadoopVer = null; private QTestSetup setup = null; @@ -554,9 +556,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, this.clusterType = clusterType; HadoopShims shims = ShimLoader.getHadoopShims(); - setupFileSystem(shims); - setup = new QTestSetup(); setup.preTest(conf); @@ -606,11 +606,6 @@ private void setupFileSystem(HadoopShims shims) throws IOException { dfs = shims.getMiniDfs(conf, numDataNodes, true, null); fs = dfs.getFileSystem(); - - // set up the java key provider for encrypted hdfs cluster - hes = shims.createHdfsEncryptionShim(fs, conf); - - LOG.info("key provider is initialized"); } else { dfs = shims.getMiniDfs(conf, numDataNodes, true, null); fs = dfs.getFileSystem(); @@ -848,15 +843,15 @@ public void clearPostTestEffects() throws Exception { } public void clearKeysCreatedInTests() { - if (hes == null) { - return; - } - try { - for (String keyAlias : hes.getKeys()) { - hes.deleteKey(keyAlias); + URI uri = fs.getUri(); + if ("hdfs".equals(uri.getScheme())) { + try { + for (String keyAlias : ((DistributedFileSystem) fs).getClient().getKeyProvider().getKeys()) { + HdfsUtils.deleteKey(keyAlias, conf); + } + } catch (IOException e) { + LOG.error("Fail to clean the keys created in test due to the error", e); } - } catch (IOException e) { - LOG.error("Fail to clean the keys created in test due to the error", e); } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index f4af6f4979..db299ca32b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -63,6 +63,7 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimaps; + import org.apache.commons.cli.OptionBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -306,7 +307,7 @@ protected Configuration initialValue() { public static final Logger auditLog = LoggerFactory.getLogger( HiveMetaStore.class.getName() + ".audit"); - + private static final void logAuditEvent(String cmd) { if (cmd == null) { return; @@ -547,7 +548,7 @@ public void init() throws MetaException { cleaner.schedule(new DumpDirCleanerTask(hiveConf), cleanFreq, cleanFreq); } expressionProxy = PartFilterExprUtil.createExpressionProxy(hiveConf); - fileMetadataManager = new FileMetadataManager((ThreadLocalRawStore)this, hiveConf); + fileMetadataManager = new FileMetadataManager(this, hiveConf); } private static String addPrefix(String s) { @@ -5616,6 +5617,7 @@ private void rethrowException(Exception e) } } + @Override public int get_num_partitions_by_filter(final String dbName, final String tblName, final String filter) throws TException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 436a2fe73b..d9f93659a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -52,6 +52,8 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.collect.ImmutableMap; + import javax.jdo.JDODataStoreException; import org.apache.calcite.plan.RelOptMaterialization; @@ -62,7 +64,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.hive.common.BlobStorageUtils; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.common.ObjectPair; @@ -149,7 +150,6 @@ import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; -import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.security.UserGroupInformation; @@ -158,7 +158,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -2973,7 +2972,7 @@ public PrincipalPrivilegeSet get_privilege_set(HiveObjectType objectType, private static void copyFiles(final HiveConf conf, final FileSystem destFs, FileStatus[] srcs, final FileSystem srcFs, final Path destf, final boolean isSrcLocal, final List newFiles) - throws HiveException { + throws HiveException, IOException { final HdfsUtils.HadoopFileStatus fullDestStatus; try { @@ -3005,7 +3004,7 @@ private static void copyFiles(final HiveConf conf, final FileSystem destFs, final SessionState parentSession = SessionState.get(); for (final FileStatus srcFile : files) { final Path srcP = srcFile.getPath(); - final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs); + final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs, conf); final boolean isRenameAllowed = !needToCopy && !isSrcLocal; @@ -3276,7 +3275,7 @@ public static boolean moveFile(final HiveConf conf, Path srcf, final Path destf, destFs.copyFromLocalFile(srcf, destf); return true; } else { - if (needToCopy(srcf, destf, srcFs, destFs)) { + if (needToCopy(srcf, destf, srcFs, destFs, conf)) { //copy if across file system or encryption zones. LOG.debug("Copying source " + srcf + " to " + destf + " because HDFS encryption zones are different."); return FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf, @@ -3401,23 +3400,15 @@ static private HiveException getHiveException(Exception e, String msg, String lo * TODO- consider if need to do this for different file authority. * @throws HiveException */ - static protected boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs) throws HiveException { + static protected boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, FileSystem destFs, HiveConf conf) throws HiveException, IOException { //Check if different FileSystems if (!FileUtils.equalsFileSystem(srcFs, destFs)) { return true; } //Check if different encryption zones - HadoopShims.HdfsEncryptionShim srcHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(srcFs); - HadoopShims.HdfsEncryptionShim destHdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(destFs); - try { - return srcHdfsEncryptionShim != null - && destHdfsEncryptionShim != null - && (srcHdfsEncryptionShim.isPathEncrypted(srcf) || destHdfsEncryptionShim.isPathEncrypted(destf)) - && !srcHdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf, destHdfsEncryptionShim); - } catch (IOException e) { - throw new HiveException(e); - } + return (HdfsUtils.isPathEncrypted(srcf, conf) || HdfsUtils.isPathEncrypted(destf,conf)) + && !HdfsUtils.arePathsOnSameEncryptionZone(srcf, destf, conf); } /** @@ -3432,9 +3423,10 @@ static protected boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, Fil * @param newFiles if this is non-null, a list of files that were created as a result of this * move will be returned. * @throws HiveException + * @throws IOException */ static protected void copyFiles(HiveConf conf, Path srcf, Path destf, - FileSystem fs, boolean isSrcLocal, boolean isAcid, List newFiles) throws HiveException { + FileSystem fs, boolean isSrcLocal, boolean isAcid, List newFiles) throws HiveException, IOException { try { // create the destination if it does not exist if (!fs.exists(destf)) { @@ -4025,6 +4017,7 @@ public void cancelDelegationToken(String tokenStrForm) /** * @deprecated use {@link #compact2(String, String, String, String, Map)} */ + @Deprecated public void compact(String dbname, String tableName, String partName, String compactType, Map tblproperties) throws HiveException { compact2(dbname, tableName, partName, compactType, tblproperties); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 61f6a7c4ff..d64121640b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -61,8 +61,6 @@ import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.shims.HadoopShims; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.ql.stats.StatsUtils; import org.apache.thrift.TException; @@ -196,7 +194,7 @@ public void truncateTable(String dbName, String tableName, List partName Collections.sort(tableNames); return tableNames; } - + @Override public List getTableMeta(String dbPatterns, String tablePatterns, List tableTypes) throws MetaException { @@ -236,7 +234,7 @@ public void truncateTable(String dbName, String tableName, List partName } return tableMetas; } - + private boolean matchesAny(String string, List matchers) { for (Matcher matcher : matchers) { if (matcher.reset(string).matches()) { @@ -570,9 +568,7 @@ private void truncateTempTable(org.apache.hadoop.hive.metastore.api.Table table) Path location = new Path(table.getSd().getLocation()); FileSystem fs = location.getFileSystem(conf); - HadoopShims.HdfsEncryptionShim shim - = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); - if (!shim.isPathEncrypted(location)) { + if (!HdfsUtils.isPathEncrypted(location, conf)) { HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location); FileStatus targetStatus = fs.getFileStatus(location); String targetGroup = targetStatus == null ? null : targetStatus.getGroup(); @@ -655,7 +651,7 @@ private void dropTempTable(org.apache.hadoop.hive.metastore.api.Table table, boo public static Map getTempTablesForDatabase(String dbName) { return getTempTables().get(dbName); } - + public static Map> getTempTables() { SessionState ss = SessionState.get(); if (ss == null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 2fbdd70957..dc8f9455a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; @@ -2309,19 +2310,11 @@ private void getMetaData(QB qb, ReadEntity parentInput) * @throws HiveException If an error occurs while checking for encryption */ private boolean isPathEncrypted(Path path) throws HiveException { - try { - HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(path.getFileSystem(conf)); - if (hdfsEncryptionShim != null) { - if (hdfsEncryptionShim.isPathEncrypted(path)) { - return true; - } - } + return HdfsUtils.isPathEncrypted(path, conf); } catch (Exception e) { - throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e); + throw new HiveException("Unable to determine if " + path + " is encrypted: " + e, e); } - - return false; } /** @@ -2333,18 +2326,11 @@ private boolean isPathEncrypted(Path path) throws HiveException { * @throws HiveException If an error occurs while comparing key strengths. */ private int comparePathKeyStrength(Path p1, Path p2) throws HiveException { - HadoopShims.HdfsEncryptionShim hdfsEncryptionShim; - - hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim(); - if (hdfsEncryptionShim != null) { - try { - return hdfsEncryptionShim.comparePathKeyStrength(p1, p2); - } catch (Exception e) { - throw new HiveException("Unable to compare key strength for " + p1 + " and " + p2 + " : " + e, e); - } + try { + return HdfsUtils.comparePathKeyStrength(p1, p2, conf); + } catch (IOException e) { + throw new HiveException("Unable to compare key strength for " + p1 + " and " + p2 + " : " + e, e); } - - return 0; // Non-encrypted path (or equals strength) } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java index 412bcd37f6..4b185b08e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java @@ -98,11 +98,7 @@ public static CommandProcessor getForHiveCommandInternal(String[] cmd, HiveConf case RELOAD: return new ReloadProcessor(); case CRYPTO: - try { - return new CryptoProcessor(SessionState.get().getHdfsEncryptionShim(), conf); - } catch (HiveException e) { - throw new SQLException("Fail to start the command processor due to the exception: ", e); - } + return new CryptoProcessor(conf); default: throw new AssertionError("Unknown HiveCommand " + hiveCommand); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java index 1acdc9505e..0396c9f0f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java @@ -28,9 +28,9 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.shims.HadoopShims; import java.io.IOException; import java.util.Arrays; @@ -42,18 +42,15 @@ public class CryptoProcessor implements CommandProcessor { public static final Logger LOG = LoggerFactory.getLogger(CryptoProcessor.class.getName()); - private HadoopShims.HdfsEncryptionShim encryptionShim; + private final Options CREATE_KEY_OPTIONS; + private final Options DELETE_KEY_OPTIONS; + private final Options CREATE_ZONE_OPTIONS; - private Options CREATE_KEY_OPTIONS; - private Options DELETE_KEY_OPTIONS; - private Options CREATE_ZONE_OPTIONS; + private final int DEFAULT_BIT_LENGTH = 128; - private int DEFAULT_BIT_LENGTH = 128; + private final HiveConf conf; - private HiveConf conf; - - public CryptoProcessor(HadoopShims.HdfsEncryptionShim encryptionShim, HiveConf conf) { - this.encryptionShim = encryptionShim; + public CryptoProcessor (HiveConf conf) { this.conf = conf; CREATE_KEY_OPTIONS = new Options(); @@ -93,10 +90,6 @@ public CommandProcessorResponse run(String command) throws CommandNeedRetryExcep return returnErrorResponse("Command arguments are empty."); } - if (encryptionShim == null) { - return returnErrorResponse("Hadoop encryption shim is not initialized."); - } - String action = args[0]; String params[] = Arrays.copyOfRange(args, 1, args.length); @@ -130,7 +123,7 @@ private void createEncryptionKey(String[] params) throws Exception { String bitLength = args.getOptionValue("bitLength", Integer.toString(DEFAULT_BIT_LENGTH)); try { - encryptionShim.createKey(keyName, new Integer(bitLength)); + HdfsUtils.createKey(keyName, new Integer(bitLength), conf); } catch (Exception e) { throw new Exception("Cannot create encryption key: " + e.getMessage()); } @@ -149,13 +142,8 @@ private void createEncryptionZone(String[] params) throws Exception { String keyName = args.getOptionValue("keyName"); Path cryptoZone = new Path(args.getOptionValue("path")); - if (cryptoZone == null) { - throw new Exception("Cannot create encryption zone: Invalid path '" - + args.getOptionValue("path") + "'"); - } - try { - encryptionShim.createEncryptionZone(cryptoZone, keyName); + HdfsUtils.createEncryptionZone(cryptoZone, keyName, conf); } catch (IOException e) { throw new Exception("Cannot create encryption zone: " + e.getMessage()); } @@ -174,7 +162,7 @@ private void deleteEncryptionKey(String[] params) throws Exception { String keyName = args.getOptionValue("keyName"); try { - encryptionShim.deleteKey(keyName); + HdfsUtils.deleteKey(keyName, conf); } catch (IOException e) { throw new Exception("Cannot delete encryption key: " + e.getMessage()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 6dece05c3a..112a8ce641 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -45,6 +45,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -85,16 +87,11 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext.CLIENT_TYPE; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl; import org.apache.hadoop.hive.ql.util.ResourceDownloader; -import org.apache.hadoop.hive.shims.HadoopShims; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; /** * SessionState encapsulates common data associated with a session. @@ -226,12 +223,7 @@ private SparkSession sparkSession; - /** - * Gets information about HDFS encryption - */ - private Map hdfsEncryptionShims = Maps.newHashMap(); - - /** + /* * Lineage state. */ LineageState ls; @@ -291,7 +283,7 @@ private String atsDomainId; - private List cleanupItems = new LinkedList(); + private final List cleanupItems = new LinkedList(); /** * Get the lineage state stored in this session. @@ -483,33 +475,6 @@ public HiveTxnManager setTxnMgr(HiveTxnManager mgr) { txnMgr = mgr; return tmp; } - public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() throws HiveException { - try { - return getHdfsEncryptionShim(FileSystem.get(sessionConf)); - } - catch(HiveException hiveException) { - throw hiveException; - } - catch(Exception exception) { - throw new HiveException(exception); - } - } - - public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim(FileSystem fs) throws HiveException { - if (!hdfsEncryptionShims.containsKey(fs.getUri())) { - try { - if ("hdfs".equals(fs.getUri().getScheme())) { - hdfsEncryptionShims.put(fs.getUri(), ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, sessionConf)); - } else { - LOG.info("Could not get hdfsEncryptionShim, it is only applicable to hdfs filesystem."); - } - } catch (Exception e) { - throw new HiveException(e); - } - } - - return hdfsEncryptionShims.get(fs.getUri()); - } // SessionState is not available in runtime and Hive.get().getConf() is not safe to call private static class SessionStates { diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index e9445eb11d..77bca6d2ef 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -42,7 +42,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.key.KeyProvider; -import org.apache.hadoop.crypto.key.KeyProvider.Options; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.DefaultFileAccess; @@ -64,7 +63,6 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.io.LongWritable; @@ -1154,208 +1152,6 @@ public boolean runDistCp(List srcPaths, Path dst, Configuration conf) thro } } - private static Boolean hdfsEncryptionSupport; - - public static boolean isHdfsEncryptionSupported() { - if (hdfsEncryptionSupport == null) { - Method m = null; - - try { - m = HdfsAdmin.class.getMethod("getEncryptionZoneForPath", Path.class); - } catch (NoSuchMethodException e) { - // This version of Hadoop does not support HdfsAdmin.getEncryptionZoneForPath(). - // Hadoop 2.6.0 introduces this new method. - } - - hdfsEncryptionSupport = (m != null); - } - - return hdfsEncryptionSupport; - } - - public class HdfsEncryptionShim implements HadoopShims.HdfsEncryptionShim { - private final String HDFS_SECURITY_DEFAULT_CIPHER = "AES/CTR/NoPadding"; - - /** - * Gets information about HDFS encryption zones - */ - private HdfsAdmin hdfsAdmin = null; - - /** - * Used to compare encryption key strengths. - */ - private KeyProvider keyProvider = null; - - private final Configuration conf; - - public HdfsEncryptionShim(URI uri, Configuration conf) throws IOException { - DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.get(uri, conf); - - this.conf = conf; - this.keyProvider = dfs.getClient().getKeyProvider(); - this.hdfsAdmin = new HdfsAdmin(uri, conf); - } - - @Override - public boolean isPathEncrypted(Path path) throws IOException { - Path fullPath; - if (path.isAbsolute()) { - fullPath = path; - } else { - fullPath = path.getFileSystem(conf).makeQualified(path); - } - if(!"hdfs".equalsIgnoreCase(path.toUri().getScheme())) { - return false; - } - try { - return (hdfsAdmin.getEncryptionZoneForPath(fullPath) != null); - } catch (FileNotFoundException fnfe) { - LOG.debug("Failed to get EZ for non-existent path: "+ fullPath, fnfe); - return false; - } - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { - return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1), - hdfsAdmin.getEncryptionZoneForPath(path2)); - } - - private boolean equivalentEncryptionZones(EncryptionZone zone1, EncryptionZone zone2) { - if (zone1 == null && zone2 == null) { - return true; - } else if (zone1 == null || zone2 == null) { - return false; - } - - return zone1.equals(zone2); - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, - HadoopShims.HdfsEncryptionShim encryptionShim2) throws IOException { - if (!(encryptionShim2 instanceof Hadoop23Shims.HdfsEncryptionShim)) { - LOG.warn("EncryptionShim for path2 (" + path2 + ") is of unexpected type: " + encryptionShim2.getClass() - + ". Assuming path2 is on the same EncryptionZone as path1(" + path1 + ")."); - return true; - } - - return equivalentEncryptionZones(hdfsAdmin.getEncryptionZoneForPath(path1), - ((HdfsEncryptionShim)encryptionShim2).hdfsAdmin.getEncryptionZoneForPath(path2)); - } - - /** - * Compares two encryption key strengths. - * - * @param path1 First path to compare - * @param path2 Second path to compare - * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. - * @throws IOException If an error occurred attempting to get key metadata - */ - @Override - public int comparePathKeyStrength(Path path1, Path path2) throws IOException { - EncryptionZone zone1, zone2; - - zone1 = hdfsAdmin.getEncryptionZoneForPath(path1); - zone2 = hdfsAdmin.getEncryptionZoneForPath(path2); - - if (zone1 == null && zone2 == null) { - return 0; - } else if (zone1 == null) { - return -1; - } else if (zone2 == null) { - return 1; - } - - return compareKeyStrength(zone1, zone2); - } - - @Override - public void createEncryptionZone(Path path, String keyName) throws IOException { - hdfsAdmin.createEncryptionZone(path, keyName); - } - - @Override - public void createKey(String keyName, int bitLength) - throws IOException, NoSuchAlgorithmException { - - checkKeyProvider(); - - if (keyProvider.getMetadata(keyName) == null) { - final KeyProvider.Options options = new Options(this.conf); - options.setCipher(HDFS_SECURITY_DEFAULT_CIPHER); - options.setBitLength(bitLength); - keyProvider.createKey(keyName, options); - keyProvider.flush(); - } else { - throw new IOException("key '" + keyName + "' already exists"); - } - } - - @Override - public void deleteKey(String keyName) throws IOException { - checkKeyProvider(); - - if (keyProvider.getMetadata(keyName) != null) { - keyProvider.deleteKey(keyName); - keyProvider.flush(); - } else { - throw new IOException("key '" + keyName + "' does not exist."); - } - } - - @Override - public List getKeys() throws IOException { - checkKeyProvider(); - return keyProvider.getKeys(); - } - - private void checkKeyProvider() throws IOException { - if (keyProvider == null) { - throw new IOException("HDFS security key provider is not configured on your server."); - } - } - - /** - * Compares two encryption key strengths. - * - * @param zone1 First EncryptionZone to compare - * @param zone2 Second EncryptionZone to compare - * @return 1 if zone1 is stronger; 0 if zones are equal; -1 if zone1 is weaker. - * @throws IOException If an error occurred attempting to get key metadata - */ - private int compareKeyStrength(EncryptionZone zone1, EncryptionZone zone2) throws IOException { - - // zone1, zone2 should already have been checked for nulls. - assert zone1 != null && zone2 != null : "Neither EncryptionZone under comparison can be null."; - - CipherSuite suite1 = zone1.getSuite(); - CipherSuite suite2 = zone2.getSuite(); - - if (suite1 == null && suite2 == null) { - return 0; - } else if (suite1 == null) { - return -1; - } else if (suite2 == null) { - return 1; - } - - return Integer.compare(suite1.getAlgorithmBlockSize(), suite2.getAlgorithmBlockSize()); - } - } - - @Override - public HadoopShims.HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException { - if (isHdfsEncryptionSupported()) { - URI uri = fs.getUri(); - if ("hdfs".equals(uri.getScheme())) { - return new HdfsEncryptionShim(uri, conf); - } - } - - return new HadoopShims.NoopHdfsEncryptionShim(); - } - @Override public Path getPathWithoutSchemeAndAuthority(Path path) { return Path.getPathWithoutSchemeAndAuthority(path); diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java index 277738fac7..5e0aae100e 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java @@ -19,15 +19,17 @@ package org.apache.hadoop.hive.io; import java.io.IOException; +import java.net.URI; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.hadoop.conf.Configuration; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import com.google.common.annotations.VisibleForTesting; - -import org.apache.commons.lang.ArrayUtils; - -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProvider.Options; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; @@ -39,18 +41,20 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DistributedFileSystem; - +import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Objects; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; public class HdfsUtils { - private static final Logger LOG = LoggerFactory.getLogger("shims.HdfsUtils"); + private static final Logger LOG = LoggerFactory.getLogger("shims.HdfsUtils"); // TODO: this relies on HDFS not changing the format; we assume if we could get inode ID, this // is still going to work. Otherwise, file IDs can be turned off. Later, we should use // as public utility method in HDFS to obtain the inode-based path. @@ -236,4 +240,158 @@ AclStatus getAclStatus() { return this.aclStatus; } } + + /** + * Compares two encryption key strengths. + * + * @param keyname1 Keyname to compare + * @param keyname2 Keyname to compare + * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. + * @throws IOException If an error occurred attempting to get key metadata + */ + public static int comparePathKeyStrength(Path path1, Path path2, Configuration conf) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = getEZ(path1, conf); + zone2 = getEZ(path2, conf); + + if (zone1 == null && zone2 == null) { + return 0; + } else if (zone1 == null) { + return -1; + } else if (zone2 == null) { + return 1; + } + + KeyProvider keyProvider = ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider(); + KeyProvider.Metadata meta1, meta2; + + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + + meta1 = keyProvider.getMetadata(zone1.getKeyName()); + meta2 = keyProvider.getMetadata(zone2.getKeyName()); + + if (meta1.getBitLength() < meta2.getBitLength()) { + return -1; + } else if (meta1.getBitLength() == meta2.getBitLength()) { + return 0; + } else { + return 1; + } + } + + + /** + * create encryption zone by path and keyname + * @param path HDFS path to create encryption zone + * @param keyName keyname + * @throws IOException + */ + public static void createEncryptionZone(Path path, String keyName, Configuration conf) throws IOException { + getHDFSAdmin(path, conf).createEncryptionZone(path, keyName); + } + + /** + * Checks if two HDFS paths are on the same encrypted or unencrypted zone. + * + * @param path1 Path to HDFS file system + * @param path2 Path to HDFS file system + * @return True if both paths are in the same zone; False otherwise. + * @throws IOException If an error occurred attempting to get encryption information + */ + public static boolean arePathsOnSameEncryptionZone(Path path1, Path path2, Configuration conf) throws IOException { + EncryptionZone zone1, zone2; + + zone1 = getHDFSAdmin(path1, conf).getEncryptionZoneForPath(path1); + zone2 = getHDFSAdmin(path2, conf).getEncryptionZoneForPath(path2); + + if (zone1 == null && zone2 == null) { + return true; + } else if (zone1 == null || zone2 == null) { + return false; + } + + return zone1.equals(zone2); + } + private final static String HDFS_SECURITY_DEFAULT_CIPHER = "AES/CTR/NoPadding"; + + /** + * Creates an encryption key. + * + * @param keyName Name of the key + * @param bitLength Key encryption length in bits (128 or 256). + * @throws IOException If an error occurs while creating the encryption key + * @throws NoSuchAlgorithmException If cipher algorithm is invalid. + */ + public static void createKey(String keyName, int bitLength, Configuration conf) + throws IOException, NoSuchAlgorithmException { + + KeyProvider keyProvider = ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider(); + + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + if (keyProvider.getMetadata(keyName) == null) { + final KeyProvider.Options options = new Options(conf); + options.setCipher(HDFS_SECURITY_DEFAULT_CIPHER); + options.setBitLength(bitLength); + keyProvider.createKey(keyName, options); + keyProvider.flush(); + } else { + throw new IOException("key '" + keyName + "' already exists"); + } + } + + public static void deleteKey(String keyName, Configuration conf) throws IOException { + + KeyProvider keyProvider = ((DistributedFileSystem) FileSystem.get(conf)).getClient().getKeyProvider(); + if (keyProvider == null) { + throw new IOException("HDFS security key provider is not configured on your server."); + } + if (keyProvider.getMetadata(keyName) != null) { + keyProvider.deleteKey(keyName); + keyProvider.flush(); + } else { + throw new IOException("key '" + keyName + "' does not exist."); + } + } + + private static EncryptionZone getEZ(Path path, Configuration conf) throws IOException { + + HdfsAdmin admin = getHDFSAdmin(path, conf); + if (null == admin) { + return null; + } + + Path fullPath; + if (path.isAbsolute()) { + fullPath = path; + } else { + fullPath = path.getFileSystem(conf).makeQualified(path); + } + return admin.getEncryptionZoneForPath(fullPath); + } + + /** + * Checks if a given HDFS path is encrypted. + * + * @param path Path to HDFS file system + * @return True if it is encrypted; False otherwise. + * @throws IOException If an error occurred attempting to get encryption information + */ + + public static boolean isPathEncrypted(Path path, Configuration conf) throws IOException { + + return getEZ(path, conf) != null; + } + private static HdfsAdmin getHDFSAdmin (Path path, Configuration conf) throws IOException { + + URI uri = path.toUri(); + if(!"hdfs".equalsIgnoreCase(uri.getScheme())) { + return null; + } + return new HdfsAdmin(uri, conf); + } } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 0db54d1e51..d7e218d1c2 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -23,14 +23,11 @@ import java.net.URI; import java.nio.ByteBuffer; import java.security.AccessControlException; -import java.security.NoSuchAlgorithmException; import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.TreeMap; -import com.google.common.annotations.VisibleForTesting; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; @@ -500,139 +497,6 @@ public void checkFileAccess(FileSystem fs, FileStatus status, FsAction action) */ public boolean runDistCp(List srcPaths, Path dst, Configuration conf) throws IOException; - /** - * This interface encapsulates methods used to get encryption information from - * HDFS paths. - */ - public interface HdfsEncryptionShim { - /** - * Checks if a given HDFS path is encrypted. - * - * @param path Path to HDFS file system - * @return True if it is encrypted; False otherwise. - * @throws IOException If an error occurred attempting to get encryption information - */ - public boolean isPathEncrypted(Path path) throws IOException; - - /** - * Checks if two HDFS paths are on the same encrypted or unencrypted zone. - * - * @param path1 Path to HDFS file system - * @param path2 Path to HDFS file system - * @return True if both paths are in the same zone; False otherwise. - * @throws IOException If an error occurred attempting to get encryption information - */ - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException; - - /** - * Checks if two HDFS paths are on the same encrypted or unencrypted zone. - * - * @param path1 Path to HDFS file system - * @param path2 Path to HDFS file system - * @param encryptionShim2 The encryption-shim corresponding to path2. - * @return True if both paths are in the same zone; False otherwise. - * @throws IOException If an error occurred attempting to get encryption information - */ - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim encryptionShim2) throws IOException; - - /** - * Compares two encrypted path strengths. - * - * @param path1 HDFS path to compare. - * @param path2 HDFS path to compare. - * @return 1 if path1 is stronger; 0 if paths are equals; -1 if path1 is weaker. - * @throws IOException If an error occurred attempting to get encryption/key metadata - */ - public int comparePathKeyStrength(Path path1, Path path2) throws IOException; - - /** - * create encryption zone by path and keyname - * @param path HDFS path to create encryption zone - * @param keyName keyname - * @throws IOException - */ - @VisibleForTesting - public void createEncryptionZone(Path path, String keyName) throws IOException; - - /** - * Creates an encryption key. - * - * @param keyName Name of the key - * @param bitLength Key encryption length in bits (128 or 256). - * @throws IOException If an error occurs while creating the encryption key - * @throws NoSuchAlgorithmException If cipher algorithm is invalid. - */ - @VisibleForTesting - public void createKey(String keyName, int bitLength) - throws IOException, NoSuchAlgorithmException; - - @VisibleForTesting - public void deleteKey(String keyName) throws IOException; - - @VisibleForTesting - public List getKeys() throws IOException; - } - - /** - * This is a dummy class used when the hadoop version does not support hdfs encryption. - */ - public static class NoopHdfsEncryptionShim implements HdfsEncryptionShim { - @Override - public boolean isPathEncrypted(Path path) throws IOException { - /* not supported */ - return false; - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2) throws IOException { - /* not supported */ - return true; - } - - @Override - public boolean arePathsOnSameEncryptionZone(Path path1, Path path2, HdfsEncryptionShim encryptionShim2) throws IOException { - // Not supported. - return true; - } - - @Override - public int comparePathKeyStrength(Path path1, Path path2) throws IOException { - /* not supported */ - return 0; - } - - @Override - public void createEncryptionZone(Path path, String keyName) { - /* not supported */ - } - - @Override - public void createKey(String keyName, int bitLength) { - /* not supported */ - } - - @Override - public void deleteKey(String keyName) throws IOException { - /* not supported */ - } - - @Override - public List getKeys() throws IOException{ - /* not supported */ - return null; - } - } - - /** - * Returns a new instance of the HdfsEncryption shim. - * - * @param fs A FileSystem object to HDFS - * @param conf A Configuration object - * @return A new instance of the HdfsEncryption shim. - * @throws IOException If an error occurred while creating the instance. - */ - public HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException; - public Path getPathWithoutSchemeAndAuthority(Path path); /** -- 2.13.5 (Apple Git-94)