diff --git ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
index 588e4bf..7bdf21d 100644
--- ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
+++ ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
@@ -129,6 +129,8 @@ public boolean accept(File filePath) {
private String logDirectory;
+ private String tezDirectory;
+
private String template;
private String className;
@@ -199,6 +201,14 @@ public void setTemplate(String template) {
this.template = template;
}
+ public String getTezDirectory() {
+ return tezDirectory;
+ }
+
+ public void setTezDirectory(String tezDirectory) {
+ this.tezDirectory = tezDirectory;
+ }
+
public String getTemplate() {
return template;
}
@@ -348,6 +358,7 @@ public void execute() throws BuildException {
File outDir = null;
File resultsDir = null;
File logDir = null;
+ File tezDir = null;
try {
@@ -416,6 +427,13 @@ public void execute() throws BuildException {
throw new BuildException("Log Directory " + logDir.getCanonicalPath() + " does not exist");
}
+ if (tezDirectory != null) {
+ tezDir = new File(tezDirectory);
+ if (!tezDir.exists()) {
+ tezDir.mkdirs();
+ }
+ }
+
if (resultsDirectory != null) {
resultsDir = new File(resultsDirectory);
if (!resultsDir.exists()) {
@@ -467,6 +485,7 @@ public void execute() throws BuildException {
ctx.put("resultsDir", relativePath(hiveRootDir, resultsDir));
}
ctx.put("logDir", relativePath(hiveRootDir, logDir));
+ ctx.put("tezDirectory", tezDir);
ctx.put("clusterMode", clusterMode);
ctx.put("hiveConfDir", escapePath(hiveConfDir));
ctx.put("hadoopVersion", hadoopVersion);
diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index 633ba92..bab9c4c 100644
--- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -88,7 +88,7 @@ public CheckResults(String outDir, String logDir, MiniClusterType miniMr,
String hadoopVer, String locationSubdir)
throws Exception
{
- super(outDir, logDir, miniMr, hadoopVer, "", "");
+ super(outDir, logDir, miniMr, hadoopVer, "", "", null);
this.locationSubdir = locationSubdir;
}
}
diff --git itests/qtest/pom.xml itests/qtest/pom.xml
index 5726d54..42a2d9a 100644
--- itests/qtest/pom.xml
+++ itests/qtest/pom.xml
@@ -426,6 +426,7 @@
logFile="${project.build.directory}/testparseneggen.log"
hadoopVersion="${active.hadoop.version}"
logDirectory="${project.build.directory}/qfile-results/negative/"
+ tezDirectory="${project.build.directory}/tez"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -442,6 +443,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestCliDriver"
logFile="${project.build.directory}/testclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -459,6 +461,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientnegative/" className="TestNegativeCliDriver"
logFile="${project.build.directory}/testnegativeclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientnegative/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -475,6 +478,7 @@
className="TestCompareCliDriver"
logFile="${project.build.directory}/testcompareclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientcompare/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -492,6 +496,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/" className="TestMinimrCliDriver"
logFile="${project.build.directory}/testminimrclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -513,6 +518,7 @@
className="TestMiniTezCliDriver"
logFile="${project.build.directory}/testminitezclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
+ tezDirectory="${project.build.directory}/tez"
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
@@ -534,6 +540,25 @@
hadoopVersion="${active.hadoop.version}"
initScript="q_test_init_for_encryption.sql"
cleanupScript="q_test_cleanup_for_encryption.sql"/>
+
+
diff --git itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
index b83543a..79a52f2 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
@@ -26,7 +26,7 @@
public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr,
AccumuloTestSetup setup, String initScript, String cleanupScript) throws Exception {
- super(outDir, logDir, miniMr, null, initScript, cleanupScript);
+ super(outDir, logDir, miniMr, null, initScript, cleanupScript, null);
setup.setupWithHiveConf(conf);
super.init();
}
diff --git itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
index 9c20f90..95ecd7b 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
@@ -44,7 +44,7 @@ public HBaseQTestUtil(
String initScript, String cleanupScript)
throws Exception {
- super(outDir, logDir, miniMr, null, initScript, cleanupScript);
+ super(outDir, logDir, miniMr, null, initScript, cleanupScript, null);
setup.preTest(conf);
this.conn = setup.getConnection();
super.init();
diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 479af32..5c180ec 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -126,6 +126,7 @@
private final String testFiles;
protected final String outDir;
protected final String logDir;
+ protected final String tezDir;
private final TreeMap qMap;
private final Set qSkipSet;
private final Set qSortSet;
@@ -237,9 +238,11 @@ public void normalizeNames(File path) throws Exception {
}
}
- public QTestUtil(String outDir, String logDir, String initScript, String cleanupScript) throws
- Exception {
- this(outDir, logDir, MiniClusterType.none, null, "0.20", initScript, cleanupScript);
+ public QTestUtil(String outDir, String logDir, String initScript,
+ String cleanupScript, String tezDirectory) throws
+ Exception {
+ this(outDir, logDir, MiniClusterType.none, null, "0.20", initScript,
+ cleanupScript, tezDirectory);
}
public String getOutputDirectory() {
@@ -287,6 +290,10 @@ public void initConf() throws Exception {
"/build/ql/test/data/warehouse/")).toString());
}
+ if (clusterType == MiniClusterType.tezlocal) {
+ conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, tezDir + "/jars");
+ }
+
// Windows paths should be converted after MiniMrShim.setupConfiguration()
// since setupConfiguration may overwrite configuration values.
if (Shell.WINDOWS) {
@@ -299,6 +306,7 @@ public void initConf() throws Exception {
tez,
spark,
encrypted,
+ tezlocal,
none;
public static MiniClusterType valueForString(String type) {
@@ -310,6 +318,8 @@ public static MiniClusterType valueForString(String type) {
return spark;
} else if (type.equals("encrypted")) {
return encrypted;
+ } else if (type.equals("tezlocal")) {
+ return tezlocal;
} else {
return none;
}
@@ -317,9 +327,9 @@ public static MiniClusterType valueForString(String type) {
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer,
- String initScript, String cleanupScript)
+ String initScript, String cleanupScript, String tezDirectory)
throws Exception {
- this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript);
+ this(outDir, logDir, clusterType, null, hadoopVer, initScript, cleanupScript, tezDirectory);
}
private String getKeyProviderURI() {
@@ -332,12 +342,17 @@ private String getKeyProviderURI() {
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
- String confDir, String hadoopVer, String initScript, String cleanupScript)
+ String confDir, String hadoopVer, String initScript,
+ String cleanupScript, String tezDirectory)
throws Exception {
+
this.outDir = outDir;
this.logDir = logDir;
+ this.tezDir = tezDirectory;
+
if (confDir != null && !confDir.isEmpty()) {
- HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml"));
+ HiveConf.setHiveSiteLocation(new URL("file://"
+ + new File(confDir).toURI().getPath() + "/hive-site.xml"));
System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
}
conf = new HiveConf(Driver.class);
@@ -349,7 +364,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
qHashQuerySet = new HashSet();
qSortNHashQuerySet = new HashSet();
qJavaVersionSpecificOutput = new HashSet();
- this.clusterType = clusterType;
+ QTestUtil.clusterType = clusterType;
HadoopShims shims = ShimLoader.getHadoopShims();
int numberOfDataNodes = 4;
@@ -376,7 +391,11 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString());
if (clusterType == MiniClusterType.tez) {
- mr = shims.getMiniTezCluster(conf, 4, uriString, 1);
+ mr = shims.getMiniTezCluster(conf, 4, uriString, 1, false,
+ tezDir + "/staging");
+ } else if (clusterType == MiniClusterType.tezlocal) {
+ mr = shims.getMiniTezCluster(conf, 4, uriString, 1, true,
+ tezDir + "/staging");
} else {
mr = shims.getMiniMrCluster(conf, 4, uriString, 1);
}
@@ -834,6 +853,15 @@ public String cliInit(String tname, boolean recreate) throws Exception {
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
Utilities.clearWorkMap();
+ if (QTestUtil.clusterType == MiniClusterType.tezlocal) {
+ conf.setBoolean("tez.local.mode", true);
+ conf.set("fs.defaultFS", "file:///");
+ conf.setBoolean("tez.runtime.optimize.local.fetch", true);
+ conf.set("tez.staging-dir", tezDir + "/staging");
+ conf.setInt("tez.am.inline.task.execution.max-tasks", 2);
+ conf.setBoolean("tez.ignore.lib.uris", true);
+ }
+
CliSessionState ss = new CliSessionState(conf);
assert ss != null;
ss.in = System.in;
diff --git ql/src/test/templates/TestCliDriver.vm ql/src/test/templates/TestCliDriver.vm
index 87eaf3d..e727cea 100644
--- ql/src/test/templates/TestCliDriver.vm
+++ ql/src/test/templates/TestCliDriver.vm
@@ -39,10 +39,12 @@ public class $className extends TestCase {
String hiveConfDir = "$hiveConfDir";
String initScript = "$initScript";
String cleanupScript = "$cleanupScript";
+ String tezDirectory = "$tezDirectory";
+
try {
String hadoopVer = "$hadoopVersion";
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, tezDirectory);
// do a one time initialization
qt.cleanUp();
diff --git ql/src/test/templates/TestCompareCliDriver.vm ql/src/test/templates/TestCompareCliDriver.vm
index fdc9565..517cecb 100644
--- ql/src/test/templates/TestCompareCliDriver.vm
+++ ql/src/test/templates/TestCompareCliDriver.vm
@@ -40,10 +40,11 @@ public class $className extends TestCase {
String hiveConfDir = "$hiveConfDir";
String initScript = "$initScript";
String cleanupScript = "$cleanupScript";
+ String tezDirectory = "$tezDirectory";
try {
String hadoopVer = "$hadoopVersion";
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, tezDirectory);
// do a one time initialization
qt.cleanUp();
@@ -107,9 +108,9 @@ public class $className extends TestCase {
}
}
TestSuite suite = new TestSuite();
-
-
-
+
+
+
#foreach ($qf in $qfiles)
#set ($fname = $qf.getName())
#set ($eidx = $fname.indexOf('.'))
@@ -121,7 +122,7 @@ public class $className extends TestCase {
suite.addTest(new $className("testCompareCliDriver_shutdown"));
return suite;
}
-
+
private Map> versionFiles = new HashMap>();
static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, "
@@ -140,7 +141,7 @@ public class $className extends TestCase {
private void runTest(String tname, String fname, String fpath) throws Exception {
final String queryDirectory = HIVE_ROOT + "$queryDir";
-
+
long startTime = System.currentTimeMillis();
try {
System.err.println("Begin query: " + fname);
diff --git ql/src/test/templates/TestNegativeCliDriver.vm ql/src/test/templates/TestNegativeCliDriver.vm
index 742044a..66b8425 100644
--- ql/src/test/templates/TestNegativeCliDriver.vm
+++ ql/src/test/templates/TestNegativeCliDriver.vm
@@ -41,7 +41,7 @@ public class $className extends TestCase {
try {
String hadoopVer = "$hadoopVersion";
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
- initScript, cleanupScript);
+ initScript, cleanupScript, null);
// do a one time initialization
qt.cleanUp();
qt.createSources();
diff --git ql/src/test/templates/TestParseNegative.vm ql/src/test/templates/TestParseNegative.vm
index 33b238e..2fba245 100755
--- ql/src/test/templates/TestParseNegative.vm
+++ ql/src/test/templates/TestParseNegative.vm
@@ -32,7 +32,7 @@ public class $className extends TestCase {
private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
private static QTestUtil qt;
-
+
static {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
@@ -42,7 +42,7 @@ public class $className extends TestCase {
try {
String hadoopVer = "$hadoopVersion";
qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, hadoopVer,
- initScript, cleanupScript);
+ initScript, cleanupScript, null);
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index b17f465..49bd481 100644
--- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -228,7 +228,8 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers,
@Override
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode, int numDir) throws IOException {
+ String nameNode, int numDir, boolean local,
+ String tezDir) throws IOException {
throw new IOException("Cannot run tez on current hadoop, Version: " + VersionInfo.getVersion());
}
@@ -494,7 +495,7 @@ public void setFullFileStatus(Configuration conf, HdfsFileStatus sourceStatus,
}
public class Hadoop20SFileStatus implements HdfsFileStatus {
- private FileStatus fileStatus;
+ private final FileStatus fileStatus;
public Hadoop20SFileStatus(FileStatus fileStatus) {
this.fileStatus = fileStatus;
}
@@ -620,28 +621,33 @@ public KerberosNameShim getKerberosNameShim(String name) throws IOException {
*/
public class KerberosNameShim implements HadoopShimsSecure.KerberosNameShim {
- private KerberosName kerberosName;
+ private final KerberosName kerberosName;
public KerberosNameShim(String name) {
kerberosName = new KerberosName(name);
}
+ @Override
public String getDefaultRealm() {
return kerberosName.getDefaultRealm();
}
+ @Override
public String getServiceName() {
return kerberosName.getServiceName();
}
+ @Override
public String getHostName() {
return kerberosName.getHostName();
}
+ @Override
public String getRealm() {
return kerberosName.getRealm();
}
+ @Override
public String getShortName() throws IOException {
return kerberosName.getShortName();
}
diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index a61c3ac..31719ca 100644
--- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -351,7 +351,23 @@ public void setupConfiguration(Configuration conf) {
*/
@Override
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode, int numDir) throws IOException {
+ String nameNode, int numDir, boolean local, String tezDir) throws IOException {
+ if (local) {
+ conf.setBoolean("tez.local.mode", true);
+ conf.set("fs.defaultFS", "file:///");
+ conf.setBoolean("tez.runtime.optimize.local.fetch", true);
+ conf.setInt("tez.am.inline.task.execution.max-tasks", 6);
+ conf.set("tez.staging-dir", tezDir);
+ conf.setBoolean("tez.ignore.lib.uris", true);
+ conf.setInt("tez.runtime.io.sort.mb", 10);
+ conf.setInt("tez.runtime.unordered.output.buffer.size-mb", 20);
+ conf.setFloat("tez.runtime.shuffle.fetch.buffer.percent", 0.2f);
+ conf.setFloat("tez.runtime.shuffle.memory.limit.percent", 0.2f);
+ conf.setFloat("tez.runtime.task.input.post-merge.buffer.percent", 0f);
+
+ return null;
+ }
+
return new MiniTezShim(conf, numberOfTaskTrackers, nameNode, numDir);
}
diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 064304c..668a715 100644
--- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -93,7 +93,7 @@ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers,
String nameNode, int numDir) throws IOException;
public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers,
- String nameNode, int numDir) throws IOException;
+ String nameNode, int numDir, boolean local, String tezDir) throws IOException;
/**
* Shim for MiniMrCluster