diff --git ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java index 588e4bf..7bdf21d 100644 --- ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java +++ ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java @@ -129,6 +129,8 @@ public boolean accept(File filePath) { private String logDirectory; + private String tezDirectory; + private String template; private String className; @@ -199,6 +201,14 @@ public void setTemplate(String template) { this.template = template; } + public String getTezDirectory() { + return tezDirectory; + } + + public void setTezDirectory(String tezDirectory) { + this.tezDirectory = tezDirectory; + } + public String getTemplate() { return template; } @@ -348,6 +358,7 @@ public void execute() throws BuildException { File outDir = null; File resultsDir = null; File logDir = null; + File tezDir = null; try { @@ -416,6 +427,13 @@ public void execute() throws BuildException { throw new BuildException("Log Directory " + logDir.getCanonicalPath() + " does not exist"); } + if (tezDirectory != null) { + tezDir = new File(tezDirectory); + if (!tezDir.exists()) { + tezDir.mkdirs(); + } + } + if (resultsDirectory != null) { resultsDir = new File(resultsDirectory); if (!resultsDir.exists()) { @@ -467,6 +485,7 @@ public void execute() throws BuildException { ctx.put("resultsDir", relativePath(hiveRootDir, resultsDir)); } ctx.put("logDir", relativePath(hiveRootDir, logDir)); + ctx.put("tezDirectory", tezDir); ctx.put("clusterMode", clusterMode); ctx.put("hiveConfDir", escapePath(hiveConfDir)); ctx.put("hadoopVersion", hadoopVersion); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java index 633ba92..bab9c4c 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java @@ -88,7 +88,7 @@ public CheckResults(String outDir, String logDir, MiniClusterType miniMr, String hadoopVer, String locationSubdir) throws Exception { - super(outDir, logDir, miniMr, hadoopVer, "", ""); + super(outDir, logDir, miniMr, hadoopVer, "", "", null); this.locationSubdir = locationSubdir; } } diff --git itests/qtest/pom.xml itests/qtest/pom.xml index 5726d54..42a2d9a 100644 --- itests/qtest/pom.xml +++ itests/qtest/pom.xml @@ -426,6 +426,7 @@ logFile="${project.build.directory}/testparseneggen.log" hadoopVersion="${active.hadoop.version}" logDirectory="${project.build.directory}/qfile-results/negative/" + tezDirectory="${project.build.directory}/tez" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -442,6 +443,7 @@ resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestCliDriver" logFile="${project.build.directory}/testclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientpositive/" + tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -459,6 +461,7 @@ resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientnegative/" className="TestNegativeCliDriver" logFile="${project.build.directory}/testnegativeclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientnegative/" + tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -475,6 +478,7 @@ className="TestCompareCliDriver" logFile="${project.build.directory}/testcompareclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientcompare/" + tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -492,6 +496,7 @@ resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestMinimrCliDriver" logFile="${project.build.directory}/testminimrclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientpositive/" + tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -513,6 +518,7 @@ className="TestMiniTezCliDriver" logFile="${project.build.directory}/testminitezclidrivergen.log" logDirectory="${project.build.directory}/qfile-results/clientpositive/" + tezDirectory="${project.build.directory}/tez" hadoopVersion="${active.hadoop.version}" initScript="q_test_init.sql" cleanupScript="q_test_cleanup.sql"/> @@ -534,6 +540,25 @@ hadoopVersion="${active.hadoop.version}" initScript="q_test_init_for_encryption.sql" cleanupScript="q_test_cleanup_for_encryption.sql"/> + + diff --git itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java index b83543a..79a52f2 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java @@ -26,7 +26,7 @@ public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr, AccumuloTestSetup setup, String initScript, String cleanupScript) throws Exception { - super(outDir, logDir, miniMr, null, initScript, cleanupScript); + super(outDir, logDir, miniMr, null, initScript, cleanupScript, null); setup.setupWithHiveConf(conf); super.init(); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java index 9c20f90..95ecd7b 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java @@ -44,7 +44,7 @@ public HBaseQTestUtil( String initScript, String cleanupScript) throws Exception { - super(outDir, logDir, miniMr, null, initScript, cleanupScript); + super(outDir, logDir, miniMr, null, initScript, cleanupScript, null); setup.preTest(conf); this.conn = setup.getConnection(); super.init(); diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 479af32..5c180ec 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -126,6 +126,7 @@ private final String testFiles; protected final String outDir; protected final String logDir; + protected final String tezDir; private final TreeMap qMap; private final Set qSkipSet; private final Set qSortSet; @@ -237,9 +238,11 @@ public void normalizeNames(File path) throws Exception { } } - public QTestUtil(String outDir, String logDir, String initScript, String cleanupScript) throws - Exception { - this(outDir, logDir, MiniClusterType.none, null, "0.20", initScript, cleanupScript); + public QTestUtil(String outDir, String logDir, String initScript, + String cleanupScript, String tezDirectory) throws + Exception { + this(outDir, logDir, MiniClusterType.none, null, "0.20", initScript, + cleanupScript, tezDirectory); } public String getOutputDirectory() { @@ -287,6 +290,10 @@ public void initConf() throws Exception { "/build/ql/test/data/warehouse/")).toString()); } + if (clusterType == MiniClusterType.tezlocal) { + conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, tezDir + "/jars"); + } + // Windows paths should be converted after MiniMrShim.setupConfiguration() // since setupConfiguration may overwrite configuration values. if (Shell.WINDOWS) { @@ -299,6 +306,7 @@ public void initConf() throws Exception { tez, spark, encrypted, + tezlocal, none; public static MiniClusterType valueForString(String type) { @@ -310,6 +318,8 @@ public static MiniClusterType valueForString(String type) { return spark; } else if (type.equals("encrypted")) { return encrypted; + } else if (type.equals("tezlocal")) { + return tezlocal; } else { return none; } @@ -317,9 +327,9 @@ public static MiniClusterType valueForString(String type) { } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer, - String initScript, String cleanupScript) + String initScript, String cleanupScript, String tezDirectory) throws Exception { - this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript); + this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript, tezDirectory); } private String getKeyProviderURI() { @@ -332,12 +342,17 @@ private String getKeyProviderURI() { } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, - String confDir, String hadoopVer, String initScript, String cleanupScript) + String confDir, String hadoopVer, String initScript, + String cleanupScript, String tezDirectory) throws Exception { + this.outDir = outDir; this.logDir = logDir; + this.tezDir = tezDirectory; + if (confDir != null && !confDir.isEmpty()) { - HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml")); + HiveConf.setHiveSiteLocation(new URL("file://" + + new File(confDir).toURI().getPath() + "/hive-site.xml")); System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation()); } conf = new HiveConf(Driver.class); @@ -349,7 +364,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, qHashQuerySet = new HashSet(); qSortNHashQuerySet = new HashSet(); qJavaVersionSpecificOutput = new HashSet(); - this.clusterType = clusterType; + QTestUtil.clusterType = clusterType; HadoopShims shims = ShimLoader.getHadoopShims(); int numberOfDataNodes = 4; @@ -376,7 +391,11 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString()); if (clusterType == MiniClusterType.tez) { - mr = shims.getMiniTezCluster(conf, 4, uriString, 1); + mr = shims.getMiniTezCluster(conf, 4, uriString, 1, false, + tezDir + "/staging"); + } else if (clusterType == MiniClusterType.tezlocal) { + mr = shims.getMiniTezCluster(conf, 4, uriString, 1, true, + tezDir + "/staging"); } else { mr = shims.getMiniMrCluster(conf, 4, uriString, 1); } @@ -834,6 +853,15 @@ public String cliInit(String tname, boolean recreate) throws Exception { HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); Utilities.clearWorkMap(); + if (QTestUtil.clusterType == MiniClusterType.tezlocal) { + conf.setBoolean("tez.local.mode", true); + conf.set("fs.defaultFS", "file:///"); + conf.setBoolean("tez.runtime.optimize.local.fetch", true); + conf.set("tez.staging-dir", tezDir + "/staging"); + conf.setInt("tez.am.inline.task.execution.max-tasks", 2); + conf.setBoolean("tez.ignore.lib.uris", true); + } + CliSessionState ss = new CliSessionState(conf); assert ss != null; ss.in = System.in; diff --git ql/src/test/templates/TestCliDriver.vm ql/src/test/templates/TestCliDriver.vm index 87eaf3d..e727cea 100644 --- ql/src/test/templates/TestCliDriver.vm +++ ql/src/test/templates/TestCliDriver.vm @@ -39,10 +39,12 @@ public class $className extends TestCase { String hiveConfDir = "$hiveConfDir"; String initScript = "$initScript"; String cleanupScript = "$cleanupScript"; + String tezDirectory = "$tezDirectory"; + try { String hadoopVer = "$hadoopVersion"; qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, - hiveConfDir, hadoopVer, initScript, cleanupScript); + hiveConfDir, hadoopVer, initScript, cleanupScript, tezDirectory); // do a one time initialization qt.cleanUp(); diff --git ql/src/test/templates/TestCompareCliDriver.vm ql/src/test/templates/TestCompareCliDriver.vm index fdc9565..517cecb 100644 --- ql/src/test/templates/TestCompareCliDriver.vm +++ ql/src/test/templates/TestCompareCliDriver.vm @@ -40,10 +40,11 @@ public class $className extends TestCase { String hiveConfDir = "$hiveConfDir"; String initScript = "$initScript"; String cleanupScript = "$cleanupScript"; + String tezDirectory = "$tezDirectory"; try { String hadoopVer = "$hadoopVersion"; qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, - hiveConfDir, hadoopVer, initScript, cleanupScript); + hiveConfDir, hadoopVer, initScript, cleanupScript, tezDirectory); // do a one time initialization qt.cleanUp(); @@ -107,9 +108,9 @@ public class $className extends TestCase { } } TestSuite suite = new TestSuite(); - - - + + + #foreach ($qf in $qfiles) #set ($fname = $qf.getName()) #set ($eidx = $fname.indexOf('.')) @@ -121,7 +122,7 @@ public class $className extends TestCase { suite.addTest(new $className("testCompareCliDriver_shutdown")); return suite; } - + private Map> versionFiles = new HashMap>(); static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " @@ -140,7 +141,7 @@ public class $className extends TestCase { private void runTest(String tname, String fname, String fpath) throws Exception { final String queryDirectory = HIVE_ROOT + "$queryDir"; - + long startTime = System.currentTimeMillis(); try { System.err.println("Begin query: " + fname); diff --git ql/src/test/templates/TestNegativeCliDriver.vm ql/src/test/templates/TestNegativeCliDriver.vm index 742044a..66b8425 100644 --- ql/src/test/templates/TestNegativeCliDriver.vm +++ ql/src/test/templates/TestNegativeCliDriver.vm @@ -41,7 +41,7 @@ public class $className extends TestCase { try { String hadoopVer = "$hadoopVersion"; qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer, - initScript, cleanupScript); + initScript, cleanupScript, null); // do a one time initialization qt.cleanUp(); qt.createSources(); diff --git ql/src/test/templates/TestParseNegative.vm ql/src/test/templates/TestParseNegative.vm index 33b238e..2fba245 100755 --- ql/src/test/templates/TestParseNegative.vm +++ ql/src/test/templates/TestParseNegative.vm @@ -32,7 +32,7 @@ public class $className extends TestCase { private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; - + static { MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); @@ -42,7 +42,7 @@ public class $className extends TestCase { try { String hadoopVer = "$hadoopVersion"; qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer, - initScript, cleanupScript); + initScript, cleanupScript, null); } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index b17f465..49bd481 100644 --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -228,7 +228,8 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, @Override public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir) throws IOException { + String nameNode, int numDir, boolean local, + String tezDir) throws IOException { throw new IOException("Cannot run tez on current hadoop, Version: " + VersionInfo.getVersion()); } @@ -494,7 +495,7 @@ public void setFullFileStatus(Configuration conf, HdfsFileStatus sourceStatus, } public class Hadoop20SFileStatus implements HdfsFileStatus { - private FileStatus fileStatus; + private final FileStatus fileStatus; public Hadoop20SFileStatus(FileStatus fileStatus) { this.fileStatus = fileStatus; } @@ -620,28 +621,33 @@ public KerberosNameShim getKerberosNameShim(String name) throws IOException { */ public class KerberosNameShim implements HadoopShimsSecure.KerberosNameShim { - private KerberosName kerberosName; + private final KerberosName kerberosName; public KerberosNameShim(String name) { kerberosName = new KerberosName(name); } + @Override public String getDefaultRealm() { return kerberosName.getDefaultRealm(); } + @Override public String getServiceName() { return kerberosName.getServiceName(); } + @Override public String getHostName() { return kerberosName.getHostName(); } + @Override public String getRealm() { return kerberosName.getRealm(); } + @Override public String getShortName() throws IOException { return kerberosName.getShortName(); } diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index a61c3ac..31719ca 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -351,7 +351,23 @@ public void setupConfiguration(Configuration conf) { */ @Override public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir) throws IOException { + String nameNode, int numDir, boolean local, String tezDir) throws IOException { + if (local) { + conf.setBoolean("tez.local.mode", true); + conf.set("fs.defaultFS", "file:///"); + conf.setBoolean("tez.runtime.optimize.local.fetch", true); + conf.setInt("tez.am.inline.task.execution.max-tasks", 6); + conf.set("tez.staging-dir", tezDir); + conf.setBoolean("tez.ignore.lib.uris", true); + conf.setInt("tez.runtime.io.sort.mb", 10); + conf.setInt("tez.runtime.unordered.output.buffer.size-mb", 20); + conf.setFloat("tez.runtime.shuffle.fetch.buffer.percent", 0.2f); + conf.setFloat("tez.runtime.shuffle.memory.limit.percent", 0.2f); + conf.setFloat("tez.runtime.task.input.post-merge.buffer.percent", 0f); + + return null; + } + return new MiniTezShim(conf, numberOfTaskTrackers, nameNode, numDir); } diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 064304c..668a715 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -93,7 +93,7 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException; public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers, - String nameNode, int numDir) throws IOException; + String nameNode, int numDir, boolean local, String tezDir) throws IOException; /** * Shim for MiniMrCluster