diff --git build-common.xml build-common.xml index e68ecea..3049ec2 100644 --- build-common.xml +++ build-common.xml @@ -443,7 +443,7 @@ - + diff --git build.properties build.properties index 2d293a6..3d10832 100644 --- build.properties +++ build.properties @@ -131,6 +131,9 @@ datanucleus.repo=http://www.datanucleus.org/downloads/maven2 # JVM arguments jvm.args=-XX:-UseSplitVerifier +# junit jvm args +junit.jvm.args=-XX:-UseSplitVerifier -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=128M + # # Eclipse Properties # diff --git ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java index 8d7d6bf..d3dfc0b 100644 --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java @@ -79,7 +79,6 @@ import org.apache.hadoop.hive.serde2.thrift.test.Complex; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; @@ -117,7 +116,7 @@ private FileSystem fs; protected final boolean overWrite; private CliDriver cliDriver; - private MiniMRCluster mr = null; + private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; private boolean miniMr = false; private String hadoopVer = null; @@ -214,36 +213,6 @@ public void initConf() throws Exception { convertPathsFromWindowsToHdfs(); } - if (miniMr) { - assert dfs != null; - assert mr != null; - // set fs.default.name to the uri of mini-dfs - String dfsUriString = getHdfsUriString(dfs.getFileSystem().getUri().toString()); - conf.setVar(HiveConf.ConfVars.HADOOPFS, dfsUriString); - // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs - conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, - (new Path(dfsUriString, - "/build/ql/test/data/warehouse/")).toString()); - int port = 0; - - try { - // Hadoop20 MiniMRCluster will return a proper port. - // Hadoop23 MiniMRCluster does not implement this method so use the default RM port. - port = mr.getJobTrackerPort(); - } catch (UnsupportedOperationException e) { - String address = - StringUtils.substringAfterLast(conf.get("yarn.resourcemanager.address"), ":"); - - if (StringUtils.isBlank(address)) { - throw new IllegalArgumentException("Invalid YARN resource manager port."); - } - - port = Integer.parseInt(address); - } - - ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, - "localhost:" + port); - } } private void convertPathsFromWindowsToHdfs() { @@ -294,7 +263,7 @@ public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer) if (miniMr) { dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null); FileSystem fs = dfs.getFileSystem(); - mr = new MiniMRCluster(4, getHdfsUriString(fs.getUri().toString()), 1); + mr = ShimLoader.getHadoopShims().getMiniMrCluster(conf, 4, getHdfsUriString(fs.getUri().toString()), 1); } initConf(); diff --git shims/ivy.xml shims/ivy.xml index a18634b..9126abe 100644 --- shims/ivy.xml +++ shims/ivy.xml @@ -91,6 +91,36 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java index ccb714a..06abbb4 100644 --- shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.InputFormat; @@ -100,6 +101,38 @@ public void setTmpFiles(String prop, String files) { // gone in 20+ } + + /** + * Returns a shim to wrap MiniMrCluster + */ + public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, + String nameNode, int numDir) throws IOException { + return new MiniMrShim(new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir)); + } + + /** + * Shim for MiniMrCluster + */ + public class MiniMrShim implements HadoopShims.MiniMrShim { + + private final MiniMRCluster mr; + + public MiniMrShim(MiniMRCluster mr) { + this.mr = mr; + } + + @Override + public int getJobTrackerPort() throws UnsupportedOperationException { + return mr.getJobTrackerPort(); + } + + @Override + public void shutdown() throws IOException { + mr.shutdown(); + } + + } + public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf, int numDataNodes, boolean format, diff --git shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index b7515a5..614e4b1 100644 --- shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -26,6 +26,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hive.shims.HadoopShimsSecure; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.TaskLogServlet; @@ -119,4 +121,66 @@ public long getDefaultBlockSize(FileSystem fs, Path path) { public short getDefaultReplication(FileSystem fs, Path path) { return fs.getDefaultReplication(); } + + /** + * Returns a shim to wrap MiniMrCluster + */ + public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, + String nameNode, int numDir) throws IOException { + return new MiniMrShim(new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir)); + } + + /** + * Shim for MiniMrCluster + */ + public class MiniMrShim implements HadoopShims.MiniMrShim { + + private final MiniMRCluster mr; + + public MiniMrShim(MiniMRCluster mr) { + this.mr = mr; + } + + @Override + public int getJobTrackerPort() throws UnsupportedOperationException { + return mr.getJobTrackerPort(); + } + + @Override + public void shutdown() throws IOException { + mr.shutdown(); + } + + } + + // Don't move this code to the parent class. There's a binary + // incompatibility between hadoop 1 and 2 wrt MiniDFSCluster and we + // need to have two different shim classes even though they are + // exactly the same. + public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf, + int numDataNodes, + boolean format, + String[] racks) throws IOException { + return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks)); + } + + /** + * MiniDFSShim. + * + */ + public class MiniDFSShim implements HadoopShims.MiniDFSShim { + private final MiniDFSCluster cluster; + + public MiniDFSShim(MiniDFSCluster cluster) { + this.cluster = cluster; + } + + public FileSystem getFileSystem() throws IOException { + return cluster.getFileSystem(); + } + + public void shutdown() { + cluster.shutdown(); + } + } } diff --git shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 9a22355..ea31255 100644 --- shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -21,13 +21,17 @@ import java.lang.Integer; import java.net.MalformedURLException; import java.net.URL; +import java.util.Map; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hive.shims.HadoopShims.JobTrackerState; import org.apache.hadoop.hive.shims.HadoopShimsSecure; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.Job; @@ -132,4 +136,89 @@ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration co throws IOException { return Trash.moveToAppropriateTrash(fs, path, conf); } + + /** + * Returns a shim to wrap MiniMrCluster + */ + public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, + String nameNode, int numDir) throws IOException { + + JobConf jConf = new JobConf(conf); + jConf.set("yarn.scheduler.capacity.root.queues", "default"); + jConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); + + MiniMRCluster mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf); + jConf = mr.createJobConf(); + + for (Map.Entry pair: jConf) { + if (conf.get(pair.getKey()) == null) { + conf.set(pair.getKey(), pair.getValue()); + } + } + + return new MiniMrShim(mr, conf); + } + + /** + * Shim for MiniMrCluster + */ + public class MiniMrShim implements HadoopShims.MiniMrShim { + + private final MiniMRCluster mr; + private final Configuration conf; + + public MiniMrShim(MiniMRCluster mr, Configuration conf) { + this.mr = mr; + this.conf = conf; + } + + @Override + public int getJobTrackerPort() throws UnsupportedOperationException { + String address = conf.get("yarn.resourcemanager.address"); + address = StringUtils.substringAfterLast(address, ":"); + + if (StringUtils.isBlank(address)) { + throw new IllegalArgumentException("Invalid YARN resource manager port."); + } + + return Integer.parseInt(address); + } + + @Override + public void shutdown() throws IOException { + mr.shutdown(); + } + + } + + // Don't move this code to the parent class. There's a binary + // incompatibility between hadoop 1 and 2 wrt MiniDFSCluster and we + // need to have two different shim classes even though they are + // exactly the same. + public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf, + int numDataNodes, + boolean format, + String[] racks) throws IOException { + return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks)); + } + + /** + * MiniDFSShim. + * + */ + public class MiniDFSShim implements HadoopShims.MiniDFSShim { + private final MiniDFSCluster cluster; + + public MiniDFSShim(MiniDFSCluster cluster) { + this.cluster = cluster; + } + + public FileSystem getFileSystem() throws IOException { + return cluster.getFileSystem(); + } + + public void shutdown() { + cluster.shutdown(); + } + } } diff --git shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index 2d32b07..23ff543 100644 --- shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -37,7 +37,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; import org.apache.hadoop.hive.thrift.DelegationTokenSelector; import org.apache.hadoop.http.HtmlQuoting; @@ -104,33 +103,6 @@ public void setTmpFiles(String prop, String files) { // gone in 20+ } - public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf, - int numDataNodes, - boolean format, - String[] racks) throws IOException { - return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks)); - } - - /** - * MiniDFSShim. - * - */ - public class MiniDFSShim implements HadoopShims.MiniDFSShim { - private final MiniDFSCluster cluster; - - public MiniDFSShim(MiniDFSCluster cluster) { - this.cluster = cluster; - } - - public FileSystem getFileSystem() throws IOException { - return cluster.getFileSystem(); - } - - public void shutdown() { - cluster.shutdown(); - } - } - /** * We define this function here to make the code compatible between * hadoop 0.17 and hadoop 0.20. diff --git shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java index bdb2500..7a58e58 100644 --- shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -113,6 +113,20 @@ String getTaskAttemptLogUrl(JobConf conf, long getAccessTime(FileStatus file); /** + * Returns a shim to wrap MiniMrCluster + */ + public MiniMrShim getMiniMrCluster(Configuration conf, int numberOfTaskTrackers, + String nameNode, int numDir) throws IOException; + + /** + * Shim for MiniMrCluster + */ + public interface MiniMrShim { + public int getJobTrackerPort() throws UnsupportedOperationException; + public void shutdown() throws IOException; + } + + /** * Returns a shim to wrap MiniDFSCluster. This is necessary since this class * was moved from org.apache.hadoop.dfs to org.apache.hadoop.hdfs */