diff --git build-common.xml build-common.xml
index e68ecea..3049ec2 100644
--- build-common.xml
+++ build-common.xml
@@ -443,7 +443,7 @@
-
+
diff --git build.properties build.properties
index 2d293a6..3d10832 100644
--- build.properties
+++ build.properties
@@ -131,6 +131,9 @@ datanucleus.repo=http://www.datanucleus.org/downloads/maven2
# JVM arguments
jvm.args=-XX:-UseSplitVerifier
+# junit jvm args
+junit.jvm.args=-XX:-UseSplitVerifier -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=128M
+
#
# Eclipse Properties
#
diff --git ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
index 8d7d6bf..d3dfc0b 100644
--- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
+++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -79,7 +79,6 @@
import org.apache.hadoop.hive.serde2.thrift.test.Complex;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
@@ -117,7 +116,7 @@
private FileSystem fs;
protected final boolean overWrite;
private CliDriver cliDriver;
- private MiniMRCluster mr = null;
+ private HadoopShims.MiniMrShim mr = null;
private HadoopShims.MiniDFSShim dfs = null;
private boolean miniMr = false;
private String hadoopVer = null;
@@ -214,36 +213,6 @@ public void initConf() throws Exception {
convertPathsFromWindowsToHdfs();
}
- if (miniMr) {
- assert dfs != null;
- assert mr != null;
- // set fs.default.name to the uri of mini-dfs
- String dfsUriString = getHdfsUriString(dfs.getFileSystem().getUri().toString());
- conf.setVar(HiveConf.ConfVars.HADOOPFS, dfsUriString);
- // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs
- conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
- (new Path(dfsUriString,
- "/build/ql/test/data/warehouse/")).toString());
- int port = 0;
-
- try {
- // Hadoop20 MiniMRCluster will return a proper port.
- // Hadoop23 MiniMRCluster does not implement this method so use the default RM port.
- port = mr.getJobTrackerPort();
- } catch (UnsupportedOperationException e) {
- String address =
- StringUtils.substringAfterLast(conf.get("yarn.resourcemanager.address"), ":");
-
- if (StringUtils.isBlank(address)) {
- throw new IllegalArgumentException("Invalid YARN resource manager port.");
- }
-
- port = Integer.parseInt(address);
- }
-
- ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf,
- "localhost:" + port);
- }
}
private void convertPathsFromWindowsToHdfs() {
@@ -294,7 +263,7 @@ public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer)
if (miniMr) {
dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
FileSystem fs = dfs.getFileSystem();
- mr = new MiniMRCluster(4, getHdfsUriString(fs.getUri().toString()), 1);
+ mr = ShimLoader.getHadoopShims().getMiniMrCluster(conf, 4, getHdfsUriString(fs.getUri().toString()), 1);
}
initConf();
diff --git shims/ivy.xml shims/ivy.xml
index a18634b..9126abe 100644
--- shims/ivy.xml
+++ shims/ivy.xml
@@ -91,6 +91,36 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
index ccb714a..06abbb4 100644
--- shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
+++ shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
@@ -45,6 +45,7 @@
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.InputFormat;
@@ -100,6 +101,38 @@ public void setTmpFiles(String prop, String files) {
// gone in 20+
}
+
+ /**
+ * Returns a shim to wrap MiniMrCluster
+ */
+ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers,
+ String nameNode, int numDir) throws IOException {
+ return new MiniMrShim(new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir));
+ }
+
+ /**
+ * Shim for MiniMrCluster
+ */
+ public class MiniMrShim implements HadoopShims.MiniMrShim {
+
+ private final MiniMRCluster mr;
+
+ public MiniMrShim(MiniMRCluster mr) {
+ this.mr = mr;
+ }
+
+ @Override
+ public int getJobTrackerPort() throws UnsupportedOperationException {
+ return mr.getJobTrackerPort();
+ }
+
+ @Override
+ public void shutdown() throws IOException {
+ mr.shutdown();
+ }
+
+ }
+
public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
int numDataNodes,
boolean format,
diff --git shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index b7515a5..614e4b1 100644
--- shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -26,6 +26,8 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hive.shims.HadoopShimsSecure;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TaskLogServlet;
@@ -119,4 +121,66 @@ public long getDefaultBlockSize(FileSystem fs, Path path) {
public short getDefaultReplication(FileSystem fs, Path path) {
return fs.getDefaultReplication();
}
+
+ /**
+ * Returns a shim to wrap MiniMrCluster
+ */
+ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers,
+ String nameNode, int numDir) throws IOException {
+ return new MiniMrShim(new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir));
+ }
+
+ /**
+ * Shim for MiniMrCluster
+ */
+ public class MiniMrShim implements HadoopShims.MiniMrShim {
+
+ private final MiniMRCluster mr;
+
+ public MiniMrShim(MiniMRCluster mr) {
+ this.mr = mr;
+ }
+
+ @Override
+ public int getJobTrackerPort() throws UnsupportedOperationException {
+ return mr.getJobTrackerPort();
+ }
+
+ @Override
+ public void shutdown() throws IOException {
+ mr.shutdown();
+ }
+
+ }
+
+ // Don't move this code to the parent class. There's a binary
+ // incompatibility between hadoop 1 and 2 wrt MiniDFSCluster and we
+ // need to have two different shim classes even though they are
+ // exactly the same.
+ public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
+ int numDataNodes,
+ boolean format,
+ String[] racks) throws IOException {
+ return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+ }
+
+ /**
+ * MiniDFSShim.
+ *
+ */
+ public class MiniDFSShim implements HadoopShims.MiniDFSShim {
+ private final MiniDFSCluster cluster;
+
+ public MiniDFSShim(MiniDFSCluster cluster) {
+ this.cluster = cluster;
+ }
+
+ public FileSystem getFileSystem() throws IOException {
+ return cluster.getFileSystem();
+ }
+
+ public void shutdown() {
+ cluster.shutdown();
+ }
+ }
}
diff --git shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 9a22355..ea31255 100644
--- shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -21,13 +21,17 @@
import java.lang.Integer;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.Map;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hive.shims.HadoopShims.JobTrackerState;
import org.apache.hadoop.hive.shims.HadoopShimsSecure;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
@@ -132,4 +136,89 @@ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration co
throws IOException {
return Trash.moveToAppropriateTrash(fs, path, conf);
}
+
+ /**
+ * Returns a shim to wrap MiniMrCluster
+ */
+ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers,
+ String nameNode, int numDir) throws IOException {
+
+ JobConf jConf = new JobConf(conf);
+ jConf.set("yarn.scheduler.capacity.root.queues", "default");
+ jConf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+
+ MiniMRCluster mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf);
+ jConf = mr.createJobConf();
+
+ for (Map.Entry pair: jConf) {
+ if (conf.get(pair.getKey()) == null) {
+ conf.set(pair.getKey(), pair.getValue());
+ }
+ }
+
+ return new MiniMrShim(mr, conf);
+ }
+
+ /**
+ * Shim for MiniMrCluster
+ */
+ public class MiniMrShim implements HadoopShims.MiniMrShim {
+
+ private final MiniMRCluster mr;
+ private final Configuration conf;
+
+ public MiniMrShim(MiniMRCluster mr, Configuration conf) {
+ this.mr = mr;
+ this.conf = conf;
+ }
+
+ @Override
+ public int getJobTrackerPort() throws UnsupportedOperationException {
+ String address = conf.get("yarn.resourcemanager.address");
+ address = StringUtils.substringAfterLast(address, ":");
+
+ if (StringUtils.isBlank(address)) {
+ throw new IllegalArgumentException("Invalid YARN resource manager port.");
+ }
+
+ return Integer.parseInt(address);
+ }
+
+ @Override
+ public void shutdown() throws IOException {
+ mr.shutdown();
+ }
+
+ }
+
+ // Don't move this code to the parent class. There's a binary
+ // incompatibility between hadoop 1 and 2 wrt MiniDFSCluster and we
+ // need to have two different shim classes even though they are
+ // exactly the same.
+ public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
+ int numDataNodes,
+ boolean format,
+ String[] racks) throws IOException {
+ return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+ }
+
+ /**
+ * MiniDFSShim.
+ *
+ */
+ public class MiniDFSShim implements HadoopShims.MiniDFSShim {
+ private final MiniDFSCluster cluster;
+
+ public MiniDFSShim(MiniDFSCluster cluster) {
+ this.cluster = cluster;
+ }
+
+ public FileSystem getFileSystem() throws IOException {
+ return cluster.getFileSystem();
+ }
+
+ public void shutdown() {
+ cluster.shutdown();
+ }
+ }
}
diff --git shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
index 2d32b07..23ff543 100644
--- shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
+++ shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
@@ -37,7 +37,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
import org.apache.hadoop.hive.thrift.DelegationTokenSelector;
import org.apache.hadoop.http.HtmlQuoting;
@@ -104,33 +103,6 @@ public void setTmpFiles(String prop, String files) {
// gone in 20+
}
- public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
- int numDataNodes,
- boolean format,
- String[] racks) throws IOException {
- return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
- }
-
- /**
- * MiniDFSShim.
- *
- */
- public class MiniDFSShim implements HadoopShims.MiniDFSShim {
- private final MiniDFSCluster cluster;
-
- public MiniDFSShim(MiniDFSCluster cluster) {
- this.cluster = cluster;
- }
-
- public FileSystem getFileSystem() throws IOException {
- return cluster.getFileSystem();
- }
-
- public void shutdown() {
- cluster.shutdown();
- }
- }
-
/**
* We define this function here to make the code compatible between
* hadoop 0.17 and hadoop 0.20.
diff --git shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
index bdb2500..7a58e58 100644
--- shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -113,6 +113,20 @@ String getTaskAttemptLogUrl(JobConf conf,
long getAccessTime(FileStatus file);
/**
+ * Returns a shim to wrap MiniMrCluster
+ */
+ public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers,
+ String nameNode, int numDir) throws IOException;
+
+ /**
+ * Shim for MiniMrCluster
+ */
+ public interface MiniMrShim {
+ public int getJobTrackerPort() throws UnsupportedOperationException;
+ public void shutdown() throws IOException;
+ }
+
+ /**
* Returns a shim to wrap MiniDFSCluster. This is necessary since this class
* was moved from org.apache.hadoop.dfs to org.apache.hadoop.hdfs
*/