diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index ddc2690..21ddf9a 100644 --- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -19,6 +19,7 @@ package org.apache.hive.jdbc.miniHS2; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -46,10 +47,13 @@ import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient; import org.apache.hive.service.cli.thrift.ThriftHttpCLIService; import org.apache.hive.service.server.HiveServer2; - -import com.google.common.io.Files; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MiniHS2 extends AbstractHiveService { + + private static final Logger LOG = LoggerFactory.getLogger(MiniHS2.class); + public static final String HS2_BINARY_MODE = "binary"; public static final String HS2_HTTP_MODE = "http"; private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; @@ -58,7 +62,7 @@ private static final String tmpDir = System.getProperty("test.tmp.dir"); private HiveServer2 hiveServer2 = null; private final File baseDir; - private final Path baseDfsDir; + private final Path baseFsDir; private MiniMrShim mr; private MiniDFSShim dfs; private MiniLlapCluster llapCluster = null; @@ -66,18 +70,19 @@ private boolean useMiniKdc = false; private final String serverPrincipal; private final boolean isMetastoreRemote; - private MiniClusterType miniClusterType = MiniClusterType.DFS_ONLY; + private final boolean cleanupLocalDirOnStartup; + private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; public enum MiniClusterType { MR, TEZ, LLAP, - DFS_ONLY; + LOCALFS_ONLY; } public static class Builder { private HiveConf hiveConf = new HiveConf(); - private MiniClusterType miniClusterType = MiniClusterType.DFS_ONLY; + private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; private boolean useMiniKdc = false; private String serverPrincipal; private String serverKeytab; @@ -86,6 +91,7 @@ private boolean usePortsFromConf = false; private String authType = "KERBEROS"; private boolean isHA = false; + private boolean cleanupLocalDirOnStartup = true; public Builder() { } @@ -131,6 +137,11 @@ public Builder withHTTPTransport(){ return this; } + public Builder cleanupLocalDirOnStartup(boolean val) { + this.cleanupLocalDirOnStartup = val; + return this; + } + public MiniHS2 build() throws Exception { if (miniClusterType == MiniClusterType.MR && useMiniKdc) { throw new IOException("Can't create secure miniMr ... yet"); @@ -141,7 +152,7 @@ public MiniHS2 build() throws Exception { hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE); } return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab, - isMetastoreRemote, usePortsFromConf, authType, isHA); + isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup); } } @@ -179,7 +190,7 @@ public boolean isUseMiniKdc() { private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc, String serverPrincipal, String serverKeytab, boolean isMetastoreRemote, - boolean usePortsFromConf, String authType, boolean isHA) throws Exception { + boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup) throws Exception { // Always use localhost for hostname as some tests like SSL CN validation ones // are tied to localhost being present in the certificate name super( @@ -196,11 +207,12 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM this.useMiniKdc = useMiniKdc; this.serverPrincipal = serverPrincipal; this.isMetastoreRemote = isMetastoreRemote; - baseDir = new File(tmpDir + "/local_base"); + this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; + baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); FileSystem fs; - if (miniClusterType != MiniClusterType.DFS_ONLY) { + if (miniClusterType != MiniClusterType.LOCALFS_ONLY) { // Initialize dfs dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null, isHA); fs = dfs.getFileSystem(); @@ -227,11 +239,18 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM } // store the config in system properties mr.setupConfiguration(getHiveConf()); - baseDfsDir = new Path(new Path(fs.getUri()), "/base"); + baseFsDir = new Path(new Path(fs.getUri()), "/base"); } else { - // This is DFS only mode, just initialize the dfs root directory. + // This is FS only mode, just initialize the dfs root directory. fs = FileSystem.getLocal(hiveConf); - baseDfsDir = new Path("file://" + baseDir.toURI().getPath()); + baseFsDir = new Path("file://" + baseDir.toURI().getPath()); + + if (cleanupLocalDirOnStartup) { + // Cleanup baseFsDir since it can be shared across tests. + LOG.info("Attempting to cleanup baseFsDir: {} while setting up MiniHS2", baseDir); + assert (baseFsDir.depth() >= 3); // Avoid "/tmp", directories closer to "/" + fs.delete(baseFsDir, true); + } } if (useMiniKdc) { hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL, serverPrincipal); @@ -242,8 +261,8 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator + "test_metastore;create=true"; - fs.mkdirs(baseDfsDir); - Path wareHouseDir = new Path(baseDfsDir, "warehouse"); + fs.mkdirs(baseFsDir); + Path wareHouseDir = new Path(baseFsDir, "warehouse"); // Create warehouse with 777, so that user impersonation has no issues. FileSystem.mkdirs(fs, wareHouseDir, FULL_PERM); @@ -259,7 +278,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, getBinaryPort()); hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, getHttpPort()); - Path scratchDir = new Path(baseDfsDir, "scratch"); + Path scratchDir = new Path(baseFsDir, "scratch"); // Create root scratchdir with write all, so that user impersonation has no issues. Utilities.createDirsWithPermission(hiveConf, scratchDir, WRITE_ALL_PERM, true); System.setProperty(HiveConf.ConfVars.SCRATCHDIR.varname, scratchDir.toString()); @@ -271,7 +290,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM } public MiniHS2(HiveConf hiveConf) throws Exception { - this(hiveConf, MiniClusterType.DFS_ONLY); + this(hiveConf, MiniClusterType.LOCALFS_ONLY); } public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType) throws Exception { @@ -281,7 +300,7 @@ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType) throws Exception public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsFromConf) throws Exception { this(hiveConf, clusterType, false, null, null, false, usePortsFromConf, - "KERBEROS", false); + "KERBEROS", false, true); } public void start(Map confOverlay) throws Exception { @@ -516,4 +535,18 @@ private void waitForStartup() throws Exception { public Service.STATE getState() { return hiveServer2.getServiceState(); } + + static File getBaseDir() { + File baseDir = new File(tmpDir + "/local_base"); + return baseDir; + } + + public static void cleanupLocalDir() throws IOException { + File baseDir = getBaseDir(); + try { + org.apache.hadoop.hive.common.FileUtils.deleteDirectory(baseDir); + } catch (FileNotFoundException e) { + // Ignore. Safe if it does not exist. + } + } } diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java index fe77667..c2ac97e 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java @@ -20,7 +20,6 @@ import java.io.File; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.junit.Test; @@ -85,7 +84,7 @@ private void setupSSLHs2() throws Exception { @Override protected MiniHS2 getNewMiniHS2() throws Exception { - return new MiniHS2(hiveConf, MiniHS2.MiniClusterType.DFS_ONLY, true); + return new MiniHS2(hiveConf, MiniHS2.MiniClusterType.LOCALFS_ONLY, true); } private void setupHs2() throws Exception { diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestServiceDiscoveryWithMiniHS2.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestServiceDiscoveryWithMiniHS2.java index 907ccb0..e8051e4 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestServiceDiscoveryWithMiniHS2.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestServiceDiscoveryWithMiniHS2.java @@ -41,7 +41,7 @@ import org.junit.Test; public class TestServiceDiscoveryWithMiniHS2 { - private static MiniHS2 miniHS2 = null; + private MiniHS2 miniHS2 = null; private static HiveConf hiveConf; private static TestingServer zkServer; private static String zkRootNamespace = "hs2test"; @@ -52,6 +52,7 @@ @BeforeClass public static void beforeTest() throws Exception { + MiniHS2.cleanupLocalDir(); zkServer = new TestingServer(); Class.forName(MiniHS2.getJdbcDriverName()); hiveConf = new HiveConf(); @@ -68,11 +69,12 @@ public static void afterTest() throws Exception { zkServer.close(); zkServer = null; } + MiniHS2.cleanupLocalDir(); } @Before public void setUp() throws Exception { - miniHS2 = new MiniHS2(hiveConf); + miniHS2 = new MiniHS2.Builder().withConf(hiveConf).cleanupLocalDirOnStartup(false).build(); } @After