diff --git ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java index 34952cc..f234390 100644 --- ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java +++ ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java @@ -203,14 +203,6 @@ public void setTemplate(String template) { this.template = template; } - public String getTezDirectory() { - return tezDirectory; - } - - public void setTezDirectory(String tezDirectory) { - this.tezDirectory = tezDirectory; - } - public String getTemplate() { return template; } @@ -377,7 +369,6 @@ public void execute() throws BuildException { File outDir = null; File resultsDir = null; File logDir = null; - File tezDir = null; try { @@ -446,13 +437,6 @@ public void execute() throws BuildException { throw new BuildException("Log Directory " + logDir.getCanonicalPath() + " does not exist"); } - if (tezDirectory != null) { - tezDir = new File(tezDirectory); - if (!tezDir.exists()) { - tezDir.mkdirs(); - } - } - if (resultsDirectory != null) { resultsDir = new File(resultsDirectory); if (!resultsDir.exists()) { @@ -533,7 +517,6 @@ public void execute() throws BuildException { ctx.put("resultsDir", relativePath(hiveRootDir, resultsDir)); } ctx.put("logDir", relativePath(hiveRootDir, logDir)); - ctx.put("tezDirectory", tezDir); ctx.put("clusterMode", clusterMode); if (hiveConfDir == null || hiveConfDir.isEmpty()) { ctx.put("hiveConfDir", ""); diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index c7bbead..adb8a71 100644 --- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -181,8 +181,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, 1, false, - baseDir.toString() + "/staging"); + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, 1); break; case MR: mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java index b374f95..1fe4f48 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java @@ -88,7 +88,7 @@ public CheckResults(String outDir, String logDir, MiniClusterType miniMr, String hadoopVer, String locationSubdir) throws Exception { - super(outDir, logDir, miniMr, null, hadoopVer, "", "", null, false); + super(outDir, logDir, miniMr, null, hadoopVer, "", "", false); this.locationSubdir = locationSubdir; } } diff --git itests/qtest/pom.xml itests/qtest/pom.xml index feb8072..bf38164 100644 --- itests/qtest/pom.xml +++ itests/qtest/pom.xml @@ -433,7 +433,6 @@ logFile="${project.build.directory}/testparseneggen.log" hadoopVersion="${active.hadoop.version}" logDirectory="${project.build.directory}/qfile-results/negative/" - tezDirectory="${project.build.directory}/tez" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -450,7 +449,6 @@ resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestCliDriver" logFile="${project.build.directory}/testclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientpositive/" - tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -468,7 +466,6 @@ resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientnegative/" className="TestNegativeCliDriver" logFile="${project.build.directory}/testnegativeclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientnegative/" - tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -485,7 +482,6 @@ className="TestCompareCliDriver" logFile="${project.build.directory}/testcompareclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientcompare/" - tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -503,7 +499,6 @@ resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestMinimrCliDriver" logFile="${project.build.directory}/testminimrclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientpositive/" - tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -525,7 +520,6 @@ className="TestMiniTezCliDriver" logFile="${project.build.directory}/testminitezclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientpositive/" - tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -548,24 +542,6 @@ initScript="q_test_init_for_encryption.sql" cleanupScript="q_test_cleanup_for_encryption.sql"/> - diff --git itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java index 79a52f2..b83543a 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java @@ -26,7 +26,7 @@ public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr, AccumuloTestSetup setup, String initScript, String cleanupScript) throws Exception { - super(outDir, logDir, miniMr, null, initScript, cleanupScript, null); + super(outDir, logDir, miniMr, null, initScript, cleanupScript); setup.setupWithHiveConf(conf); super.init(); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java index 95ecd7b..9c20f90 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java @@ -44,7 +44,7 @@ public HBaseQTestUtil( String initScript, String cleanupScript) throws Exception { - super(outDir, logDir, miniMr, null, initScript, cleanupScript, null); + super(outDir, logDir, miniMr, null, initScript, cleanupScript); setup.preTest(conf); this.conn = setup.getConnection(); super.init(); diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 97537bb..a0e0c94 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -131,7 +131,6 @@ private final String testFiles; protected final String outDir; protected final String logDir; - protected final String tezDir; private final TreeMap qMap; private final Set qSkipSet; private final Set qSortSet; @@ -249,10 +248,10 @@ public void normalizeNames(File path) throws Exception { } public QTestUtil(String outDir, String logDir, String initScript, - String cleanupScript, String tezDirectory) throws + String cleanupScript) throws Exception { this(outDir, logDir, MiniClusterType.none, null, "0.20", initScript, - cleanupScript, tezDirectory); + cleanupScript); } public String getOutputDirectory() { @@ -300,10 +299,6 @@ public void initConf() throws Exception { "/build/ql/test/data/warehouse/")).toString()); } - if (clusterType == MiniClusterType.tezlocal) { - conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, tezDir + "/jars"); - } - // Windows paths should be converted after MiniMrShim.setupConfiguration() // since setupConfiguration may overwrite configuration values. if (Shell.WINDOWS) { @@ -317,7 +312,6 @@ public void initConf() throws Exception { spark, encrypted, miniSparkOnYarn, - tezlocal, none; public static MiniClusterType valueForString(String type) { @@ -331,8 +325,6 @@ public static MiniClusterType valueForString(String type) { return encrypted; } else if (type.equals("miniSparkOnYarn")) { return miniSparkOnYarn; - } else if (type.equals("tezlocal")) { - return tezlocal; } else { return none; } @@ -340,9 +332,9 @@ public static MiniClusterType valueForString(String type) { } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer, - String initScript, String cleanupScript, String tezDirectory) + String initScript, String cleanupScript) throws Exception { - this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript, tezDirectory); + this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript); } private String getKeyProviderURI() { @@ -356,18 +348,17 @@ private String getKeyProviderURI() { public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, - String cleanupScript, String tezDirectory) throws Exception { + String cleanupScript) throws Exception { this(outDir, logDir, clusterType, confDir, - hadoopVer, initScript, cleanupScript, tezDirectory, true); + hadoopVer, initScript, cleanupScript, true); } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, - String cleanupScript, String tezDirectory, boolean withLlapIo) throws Exception { + String cleanupScript, boolean withLlapIo) throws Exception { this.outDir = outDir; this.logDir = logDir; - this.tezDir = tezDirectory; if (confDir != null && !confDir.isEmpty()) { HiveConf.setHiveSiteLocation(new URL("file://" @@ -411,11 +402,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); if (clusterType == MiniClusterType.tez) { - mr = shims.getMiniTezCluster(conf, 4, uriString, 1, false, - tezDir + "/staging"); - } else if (clusterType == MiniClusterType.tezlocal) { - mr = shims.getMiniTezCluster(conf, 4, uriString, 1, true, - tezDir + "/staging"); + mr = shims.getMiniTezCluster(conf, 4, uriString, 1); } else if (clusterType == MiniClusterType.miniSparkOnYarn) { mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); } else { @@ -891,14 +878,6 @@ public String cliInit(String tname, boolean recreate) throws Exception { HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); Utilities.clearWorkMap(conf); - if (QTestUtil.clusterType == MiniClusterType.tezlocal) { - conf.setBoolean("tez.local.mode", true); - conf.set("fs.defaultFS", "file:///"); - conf.setBoolean("tez.runtime.optimize.local.fetch", true); - conf.set("tez.staging-dir", tezDir + "/staging"); - conf.setInt("tez.am.inline.task.execution.max-tasks", 2); - conf.setBoolean("tez.ignore.lib.uris", true); - } CliSessionState ss = createSessionState(); assert ss != null; ss.in = System.in; @@ -1859,7 +1838,7 @@ public void run() { QTestUtil[] qt = new QTestUtil[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { qt[i] = new QTestUtil( - resDir, logDir, MiniClusterType.none, null, null, "0.20", "", "", false); + resDir, logDir, MiniClusterType.none, null, null, "0.20", "", false); qt[i].addFile(qfiles[i]); qt[i].clearTestSideEffects(); } diff --git ql/src/test/templates/TestCliDriver.vm ql/src/test/templates/TestCliDriver.vm index 71dbbc4..ffdef70 100644 --- ql/src/test/templates/TestCliDriver.vm +++ ql/src/test/templates/TestCliDriver.vm @@ -45,7 +45,6 @@ public class $className extends TestCase { String hiveConfDir = "$hiveConfDir"; String initScript = "$initScript"; String cleanupScript = "$cleanupScript"; - String tezDirectory = "$tezDirectory"; try { String hadoopVer = "$hadoopVersion"; @@ -53,7 +52,7 @@ public class $className extends TestCase { hiveConfDir = HIVE_ROOT + hiveConfDir; } qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, - hiveConfDir, hadoopVer, initScript, cleanupScript, tezDirectory); + hiveConfDir, hadoopVer, initScript, cleanupScript); // do a one time initialization qt.cleanUp(); diff --git ql/src/test/templates/TestCompareCliDriver.vm ql/src/test/templates/TestCompareCliDriver.vm index 49758fb..7f849e0 100644 --- ql/src/test/templates/TestCompareCliDriver.vm +++ ql/src/test/templates/TestCompareCliDriver.vm @@ -46,14 +46,13 @@ public class $className extends TestCase { String hiveConfDir = "$hiveConfDir"; String initScript = "$initScript"; String cleanupScript = "$cleanupScript"; - String tezDirectory = "$tezDirectory"; try { String hadoopVer = "$hadoopVersion"; if (!hiveConfDir.isEmpty()) { hiveConfDir = HIVE_ROOT + hiveConfDir; } qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, - hiveConfDir, hadoopVer, initScript, cleanupScript, tezDirectory); + hiveConfDir, hadoopVer, initScript, cleanupScript); // do a one time initialization qt.cleanUp(); diff --git ql/src/test/templates/TestNegativeCliDriver.vm ql/src/test/templates/TestNegativeCliDriver.vm index 5fab97e..5f8ee8e 100644 --- ql/src/test/templates/TestNegativeCliDriver.vm +++ ql/src/test/templates/TestNegativeCliDriver.vm @@ -47,7 +47,7 @@ public class $className extends TestCase { try { String hadoopVer = "$hadoopVersion"; qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer, - initScript, cleanupScript, null); + initScript, cleanupScript); // do a one time initialization qt.cleanUp(); qt.createSources(); diff --git ql/src/test/templates/TestParseNegative.vm ql/src/test/templates/TestParseNegative.vm index 3c20b84..c5e7bdf 100755 --- ql/src/test/templates/TestParseNegative.vm +++ ql/src/test/templates/TestParseNegative.vm @@ -48,7 +48,7 @@ public class $className extends TestCase { try { String hadoopVer = "$hadoopVersion"; qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer, - initScript, cleanupScript, null); + initScript, cleanupScript); } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index 5942e4b..574461b 100644 --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -231,8 +231,7 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, @Override public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir, boolean local, - String tezDir) throws IOException { + String nameNode, int numDir) throws IOException { throw new IOException("Cannot run tez on current hadoop, Version: " + VersionInfo.getVersion()); } diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 04416fe..ff00979 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -356,23 +356,7 @@ public void setupConfiguration(Configuration conf) { */ @Override public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir, boolean local, String tezDir) throws IOException { - if (local) { - conf.setBoolean("tez.local.mode", true); - conf.set("fs.defaultFS", "file:///"); - conf.setBoolean("tez.runtime.optimize.local.fetch", true); - conf.setInt("tez.am.inline.task.execution.max-tasks", 6); - conf.set("tez.staging-dir", tezDir); - conf.setBoolean("tez.ignore.lib.uris", true); - conf.setInt("tez.runtime.io.sort.mb", 10); - conf.setInt("tez.runtime.unordered.output.buffer.size-mb", 20); - conf.setFloat("tez.runtime.shuffle.fetch.buffer.percent", 0.2f); - conf.setFloat("tez.runtime.shuffle.memory.limit.percent", 0.2f); - conf.setFloat("tez.runtime.task.input.post-merge.buffer.percent", 0f); - - return null; - } - + String nameNode, int numDir) throws IOException { return new MiniTezShim(conf, numberOfTaskTrackers, nameNode, numDir); } diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 4184247..2b6f322 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -95,7 +95,7 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException; public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir, boolean local, String tezDir) throws IOException; + String nameNode, int numDir) throws IOException; public MiniMrShim getMiniSparkCluster(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException;