diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java index efbd465..f9e7801 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java @@ -35,6 +35,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.QTestUtil; +import org.apache.hadoop.hive.ql.QTestUtil.FsType; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import com.google.common.base.Splitter; import com.google.common.collect.Sets; @@ -63,6 +64,7 @@ private String initScript; private String hiveConfDir; private MiniClusterType clusterType; + private FsType fsType; // FIXME: null value is treated differently on the other end..when those filter will be // moved...this may change @@ -380,6 +382,10 @@ protected void setClusterType(MiniClusterType type) { } } + protected void setFsType(FsType fsType) { + this.fsType = fsType; + } + private String getSysPropValue(String propName) { String propValue = System.getProperty(propName); if (propValue == null || propValue.trim().length() == 0) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index 0068b95..08dba15 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -21,6 +21,7 @@ import java.net.MalformedURLException; import java.net.URL; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hadoop.hive.ql.parse.CoreParseNegative; @@ -172,7 +173,8 @@ public EncryptedHDFSCliConfig() { setCleanupScript("q_test_cleanup_for_encryption.sql"); setHiveConfDir("data/conf"); - setClusterType(MiniClusterType.encrypted); + setClusterType(MiniClusterType.mr); + setFsType(QTestUtil.FsType.encrypted_hdfs); } catch (Exception e) { throw new RuntimeException("can't construct cliconfig", e); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 0dfd727..909d7f6 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -51,7 +51,6 @@ import java.util.Comparator; import java.util.Deque; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -86,9 +85,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapItUtils; -import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; -import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Index; @@ -155,7 +152,6 @@ private String testWarehouse; private final String testFiles; - private final boolean useLocalFs; private final boolean localMode; protected final String outDir; protected final String logDir; @@ -170,6 +166,7 @@ private static final String SORT_SUFFIX = ".sorted"; private final HashSet srcTables; private final MiniClusterType clusterType; + private final FsType fsType; private ParseDriver pd; protected Hive db; protected QueryState queryState; @@ -414,14 +411,32 @@ private void createRemoteDirs() { } } + + public enum FsType { + local, + hdfs, + encrypted_hdfs, + } + public enum MiniClusterType { - mr, - tez, - spark, - encrypted, - miniSparkOnYarn, - llap, - none; + + mr(FsType.hdfs), + tez(FsType.hdfs), + spark(FsType.local), + miniSparkOnYarn(FsType.hdfs), + llap(FsType.hdfs), + none(FsType.local); + + + private final FsType defaultFsType; + + MiniClusterType(FsType defaultFsType) { + this.defaultFsType = defaultFsType; + } + + public FsType getDefaultFsType() { + return defaultFsType; + } public static MiniClusterType valueForString(String type) { if (type.equals("miniMR")) { @@ -430,8 +445,6 @@ public static MiniClusterType valueForString(String type) { return tez; } else if (type.equals("spark")) { return spark; - } else if (type.equals("encrypted")) { - return encrypted; } else if (type.equals("miniSparkOnYarn")) { return miniSparkOnYarn; } else if (type.equals("llap")) { @@ -442,6 +455,7 @@ public static MiniClusterType valueForString(String type) { } } + private String getKeyProviderURI() { // Use the target directory if it is not specified String HIVE_ROOT = AbstractCliConfig.HIVE_ROOT; @@ -473,19 +487,24 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, boolean useHBaseMetastore, boolean withLlapIo) throws Exception { this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - useHBaseMetastore, withLlapIo, false, false); + useHBaseMetastore, withLlapIo, false, null); } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, - boolean useHBaseMetastore, boolean withLlapIo, boolean localMode, boolean useLocalFs) + boolean useHBaseMetastore, boolean withLlapIo, boolean localMode, FsType fsType) throws Exception { LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," + " hadoopVer={}, initScript={}, cleanupScript={}, useHbaseMetaStore={}, withLlapIo={}," + - " localMode={}, useLocalFs={}" + " localMode={}, fsType={}" , outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - useHBaseMetastore, withLlapIo, localMode, useLocalFs); - this.useLocalFs = useLocalFs; + useHBaseMetastore, withLlapIo, localMode, fsType); + Preconditions.checkNotNull(clusterType, "ClusterType cannot be null"); + if (fsType != null) { + this.fsType = fsType; + } else { + this.fsType = clusterType.getDefaultFsType(); + } this.localMode = localMode; this.outDir = outDir; this.logDir = logDir; @@ -555,16 +574,12 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, private void setupFileSystem(HadoopShims shims) throws IOException { - if (useLocalFs) { - Preconditions - .checkState(clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap, - "useLocalFs can currently only be set for tez or llap"); - } - - if (clusterType != MiniClusterType.none && clusterType != MiniClusterType.spark) { + if (fsType == FsType.local) { + fs = FileSystem.getLocal(conf); + } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs) { int numDataNodes = 4; - if (clusterType == MiniClusterType.encrypted) { + if (fsType == FsType.encrypted_hdfs) { // Set the security key provider so that the MiniDFS cluster is initialized // with encryption conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); @@ -578,16 +593,11 @@ private void setupFileSystem(HadoopShims shims) throws IOException { LOG.info("key provider is initialized"); } else { - if (!useLocalFs) { - dfs = shims.getMiniDfs(conf, numDataNodes, true, null); - fs = dfs.getFileSystem(); - } else { - fs = FileSystem.getLocal(conf); - } + dfs = shims.getMiniDfs(conf, numDataNodes, true, null); + fs = dfs.getFileSystem(); } } else { - // Setup local file system - fs = FileSystem.getLocal(conf); + throw new IllegalArgumentException("Unknown or unhandled fsType [" + fsType + "]"); } } @@ -621,7 +631,7 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws } } else if (clusterType == MiniClusterType.miniSparkOnYarn) { mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); - } else if (clusterType == MiniClusterType.mr || clusterType == MiniClusterType.encrypted) { + } else if (clusterType == MiniClusterType.mr) { mr = shims.getMiniMrCluster(conf, 4, uriString, 1); } } @@ -900,7 +910,7 @@ public void clearTablesCreatedDuringTests() throws Exception { if(tblObj.isIndexTable()) { continue; } - db.dropTable(dbName, tblName, true, true, clusterType == MiniClusterType.encrypted); + db.dropTable(dbName, tblName, true, true, fsType == FsType.encrypted_hdfs); } else { // this table is defined in srcTables, drop all indexes on it List indexes = db.getIndexes(dbName, tblName, (short)-1); @@ -1563,7 +1573,7 @@ private void maskPatterns(Pattern[] patterns, String fname) throws Exception { boolean partialMaskWasMatched = false; Matcher matcher; while (null != (line = in.readLine())) { - if (clusterType == MiniClusterType.encrypted) { + if (fsType == FsType.encrypted_hdfs) { for (Pattern pattern : partialReservedPlanMask) { matcher = pattern.matcher(line); if (matcher.find()) {