diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index c7bbead..2ea0e37 100644 --- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -181,7 +181,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, 1, false, + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString, 1, baseDir.toString() + "/staging"); break; case MR: diff --git itests/qtest/pom.xml itests/qtest/pom.xml index feb8072..b611982 100644 --- itests/qtest/pom.xml +++ itests/qtest/pom.xml @@ -525,7 +525,7 @@ className="TestMiniTezCliDriver" logFile="${project.build.directory}/testminitezclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientpositive/" - tezDirectory="${project.build.directory}/tez" + tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -548,24 +548,6 @@ initScript="q_test_init_for_encryption.sql" cleanupScript="q_test_cleanup_for_encryption.sql"/> - diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 97537bb..b2c73b6 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -300,10 +300,6 @@ public void initConf() throws Exception { "/build/ql/test/data/warehouse/")).toString()); } - if (clusterType == MiniClusterType.tezlocal) { - conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, tezDir + "/jars"); - } - // Windows paths should be converted after MiniMrShim.setupConfiguration() // since setupConfiguration may overwrite configuration values. if (Shell.WINDOWS) { @@ -317,7 +313,6 @@ public void initConf() throws Exception { spark, encrypted, miniSparkOnYarn, - tezlocal, none; public static MiniClusterType valueForString(String type) { @@ -331,8 +326,6 @@ public static MiniClusterType valueForString(String type) { return encrypted; } else if (type.equals("miniSparkOnYarn")) { return miniSparkOnYarn; - } else if (type.equals("tezlocal")) { - return tezlocal; } else { return none; } @@ -411,10 +404,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); if (clusterType == MiniClusterType.tez) { - mr = shims.getMiniTezCluster(conf, 4, uriString, 1, false, - tezDir + "/staging"); - } else if (clusterType == MiniClusterType.tezlocal) { - mr = shims.getMiniTezCluster(conf, 4, uriString, 1, true, + mr = shims.getMiniTezCluster(conf, 4, uriString, 1, tezDir + "/staging"); } else if (clusterType == MiniClusterType.miniSparkOnYarn) { mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); @@ -891,14 +881,6 @@ public String cliInit(String tname, boolean recreate) throws Exception { HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); Utilities.clearWorkMap(conf); - if (QTestUtil.clusterType == MiniClusterType.tezlocal) { - conf.setBoolean("tez.local.mode", true); - conf.set("fs.defaultFS", "file:///"); - conf.setBoolean("tez.runtime.optimize.local.fetch", true); - conf.set("tez.staging-dir", tezDir + "/staging"); - conf.setInt("tez.am.inline.task.execution.max-tasks", 2); - conf.setBoolean("tez.ignore.lib.uris", true); - } CliSessionState ss = createSessionState(); assert ss != null; ss.in = System.in; diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index 5942e4b..0f9b11c 100644 --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -231,8 +231,7 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, @Override public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir, boolean local, - String tezDir) throws IOException { + String nameNode, int numDir, String tezDir) throws IOException { throw new IOException("Cannot run tez on current hadoop, Version: " + VersionInfo.getVersion()); } diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index cf4ef16..bf0b51f 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -356,23 +356,7 @@ public void setupConfiguration(Configuration conf) { */ @Override public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir, boolean local, String tezDir) throws IOException { - if (local) { - conf.setBoolean("tez.local.mode", true); - conf.set("fs.defaultFS", "file:///"); - conf.setBoolean("tez.runtime.optimize.local.fetch", true); - conf.setInt("tez.am.inline.task.execution.max-tasks", 6); - conf.set("tez.staging-dir", tezDir); - conf.setBoolean("tez.ignore.lib.uris", true); - conf.setInt("tez.runtime.io.sort.mb", 10); - conf.setInt("tez.runtime.unordered.output.buffer.size-mb", 20); - conf.setFloat("tez.runtime.shuffle.fetch.buffer.percent", 0.2f); - conf.setFloat("tez.runtime.shuffle.memory.limit.percent", 0.2f); - conf.setFloat("tez.runtime.task.input.post-merge.buffer.percent", 0f); - - return null; - } - + String nameNode, int numDir,String tezDir) throws IOException { return new MiniTezShim(conf, numberOfTaskTrackers, nameNode, numDir); } diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 4184247..2b2b9e1 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -95,7 +95,7 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException; public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir, boolean local, String tezDir) throws IOException; + String nameNode, int numDir, String tezDir) throws IOException; public MiniMrShim getMiniSparkCluster(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException;