diff --git data/conf/llap/tez-site.xml data/conf/llap/tez-site.xml
index 6f1b9d2..62f97c5 100644
--- data/conf/llap/tez-site.xml
+++ data/conf/llap/tez-site.xml
@@ -5,6 +5,11 @@
org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
+
+ tez.am.resource.memory.mb
+ 128
+
+
tez.am.task.max.failed.attempts
diff --git data/conf/perf-reg/tez-site.xml data/conf/perf-reg/tez-site.xml
index 940f390..c575544 100644
--- data/conf/perf-reg/tez-site.xml
+++ data/conf/perf-reg/tez-site.xml
@@ -1,5 +1,9 @@
+ tez.am.resource.memory.mb
+ 128
+
+
tez.am.dag.scheduler.class
org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
diff --git data/conf/tez/tez-site.xml data/conf/tez/tez-site.xml
index 940f390..c575544 100644
--- data/conf/tez/tez-site.xml
+++ data/conf/tez/tez-site.xml
@@ -1,5 +1,9 @@
+ tez.am.resource.memory.mb
+ 128
+
+
tez.am.dag.scheduler.class
org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index 176761f..3f99579 100644
--- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -208,7 +208,8 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM
// Initialize the execution engine based on cluster type
switch (miniClusterType) {
case TEZ:
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString);
+ // TODO: This should be making use of confDir to load configs setup for Tez, etc.
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString);
break;
case LLAP:
if (usePortsFromConf) {
@@ -216,10 +217,10 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM
}
llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null);
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString);
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString);
break;
case MR:
- mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1);
+ mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1);
break;
default:
throw new IllegalArgumentException("Unsupported cluster type " + mr);
diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index e49ecd9..15e6e93 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -628,12 +628,10 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws
conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ "/tez-site.xml"));
}
- int numTrackers;
+ int numTrackers = 2;
if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) {
llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir);
- numTrackers = 2;
} else {
- numTrackers = 4;
}
if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) {
mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local);
@@ -641,9 +639,9 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws
mr = shims.getMiniTezCluster(conf, numTrackers, uriString);
}
} else if (clusterType == MiniClusterType.miniSparkOnYarn) {
- mr = shims.getMiniSparkCluster(conf, 4, uriString, 1);
+ mr = shims.getMiniSparkCluster(conf, 2, uriString, 1);
} else if (clusterType == MiniClusterType.mr) {
- mr = shims.getMiniMrCluster(conf, 4, uriString, 1);
+ mr = shims.getMiniMrCluster(conf, 2, uriString, 1);
}
}
diff --git pom.xml pom.xml
index d8276f6..64f53df 100644
--- pom.xml
+++ pom.xml
@@ -90,6 +90,7 @@
1.0b3
3.3.0-release
+ -Xmx1024m
1.7
2.3
2.12.1
@@ -1000,7 +1001,7 @@
true
false
false
- -Xmx2048m -XX:MaxPermSize=512m
+ ${maven.test.jvm.args}
${test.tmp.dir}/conf
${basedir}/${hive.path.to.root}/conf
diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 341b1e5..df15189 100644
--- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -298,6 +298,12 @@ public MiniMrShim(Configuration conf, int numberOfTaskTrackers,
JobConf jConf = new JobConf(conf);
jConf.set("yarn.scheduler.capacity.root.queues", "default");
jConf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+ jConf.setInt(MRJobConfig.MAP_MEMORY_MB, 128);
+ jConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128);
+ jConf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128);
+ jConf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512);
+ jConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+ jConf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512);
mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf);
}
@@ -325,6 +331,9 @@ public void setupConfiguration(Configuration conf) {
for (Map.Entry pair: jConf) {
conf.set(pair.getKey(), pair.getValue());
}
+ conf.setInt(MRJobConfig.MAP_MEMORY_MB, 128);
+ conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128);
+ conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128);
}
}
@@ -390,6 +399,14 @@ public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException {
mr = new MiniTezCluster("hive", numberOfTaskTrackers);
+ conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512);
+ // Overrides values from the hive/tez-site.
+ conf.setInt("hive.tez.container.size", 128);
+ conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128);
+ conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 50);
conf.set("fs.defaultFS", nameNode);
conf.set("tez.am.log.level", "DEBUG");
conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");
@@ -421,6 +438,11 @@ public void setupConfiguration(Configuration conf) {
for (Map.Entry pair: config) {
conf.set(pair.getKey(), pair.getValue());
}
+ // Overrides values from the hive/tez-site.
+ conf.setInt("hive.tez.container.size", 128);
+ conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128);
+ conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 50);
}
}
@@ -460,6 +482,9 @@ public MiniSparkShim(Configuration conf, int numberOfTaskTrackers,
conf.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
// disable resource monitoring, although it should be off by default
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false);
+ conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 4096);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1024);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 4096);
configureImpersonation(conf);
mr.init(conf);
mr.start();
diff --git testutils/ptest2/src/main/resources/batch-exec.vm testutils/ptest2/src/main/resources/batch-exec.vm
index 2cc56ea..da82a6d 100644
--- testutils/ptest2/src/main/resources/batch-exec.vm
+++ testutils/ptest2/src/main/resources/batch-exec.vm
@@ -35,7 +35,7 @@ then
export PATH=$JAVA_HOME/bin/:$PATH
fi
export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${antEnvOpts}"
-export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}"
+export MAVEN_OPTS="-Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}"
export HADOOP_ROOT_LOGGER=INFO,console
export HADOOP_OPTS="-Dhive.log.dir=$logDir -Dhive.query.id=hadoop -Djava.io.tmpdir=$logDir/tmp"
cd $localDir/$instanceName/${repositoryName}-source || exit 1
diff --git testutils/ptest2/src/main/resources/source-prep.vm testutils/ptest2/src/main/resources/source-prep.vm
index 67e6a95..3cab8aa 100644
--- testutils/ptest2/src/main/resources/source-prep.vm
+++ testutils/ptest2/src/main/resources/source-prep.vm
@@ -23,7 +23,7 @@ then
export PATH=$JAVA_HOME/bin/:$PATH
fi
export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m ${antEnvOpts}"
-export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m ${mavenEnvOpts}"
+export MAVEN_OPTS="-Xmx1g ${mavenEnvOpts}"
cd $workingDir/
(
if [[ "$clearLibraryCache" == "true" ]]