diff --git data/conf/llap/hive-site.xml data/conf/llap/hive-site.xml
index d93f8b7..f4a9c74 100644
--- data/conf/llap/hive-site.xml
+++ data/conf/llap/hive-site.xml
@@ -40,7 +40,7 @@
hive.tez.container.size
- 128
+ 256
diff --git data/conf/llap/tez-site.xml data/conf/llap/tez-site.xml
index 6f1b9d2..8e72493 100644
--- data/conf/llap/tez-site.xml
+++ data/conf/llap/tez-site.xml
@@ -5,6 +5,11 @@
org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
+
+ tez.am.resource.memory.mb
+ 256
+
+
tez.am.task.max.failed.attempts
diff --git data/conf/perf-reg/tez-site.xml data/conf/perf-reg/tez-site.xml
index 940f390..c575544 100644
--- data/conf/perf-reg/tez-site.xml
+++ data/conf/perf-reg/tez-site.xml
@@ -1,5 +1,9 @@
+ tez.am.resource.memory.mb
+ 128
+
+
tez.am.dag.scheduler.class
org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
diff --git data/conf/tez/hive-site.xml data/conf/tez/hive-site.xml
index dbff10c..4f0fc4c 100644
--- data/conf/tez/hive-site.xml
+++ data/conf/tez/hive-site.xml
@@ -40,7 +40,7 @@
hive.tez.container.size
- 128
+ 256
diff --git data/conf/tez/tez-site.xml data/conf/tez/tez-site.xml
index 940f390..f0d2ffb 100644
--- data/conf/tez/tez-site.xml
+++ data/conf/tez/tez-site.xml
@@ -1,5 +1,9 @@
+ tez.am.resource.memory.mb
+ 256
+
+
tez.am.dag.scheduler.class
org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index ddc2690..2dbd04f 100644
--- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -209,7 +209,8 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM
// Initialize the execution engine based on cluster type
switch (miniClusterType) {
case TEZ:
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString);
+ // TODO: This should be making use of confDir to load configs setup for Tez, etc.
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString);
break;
case LLAP:
if (usePortsFromConf) {
@@ -217,10 +218,10 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM
}
llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null);
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString);
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString);
break;
case MR:
- mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1);
+ mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1);
break;
default:
throw new IllegalArgumentException("Unsupported cluster type " + mr);
diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index ff640d3..230fdaa 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -628,12 +628,10 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws
conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ "/tez-site.xml"));
}
- int numTrackers;
+ int numTrackers = 2;
if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) {
llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir);
- numTrackers = 2;
} else {
- numTrackers = 4;
}
if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) {
mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local);
@@ -641,9 +639,9 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws
mr = shims.getMiniTezCluster(conf, numTrackers, uriString);
}
} else if (clusterType == MiniClusterType.miniSparkOnYarn) {
- mr = shims.getMiniSparkCluster(conf, 4, uriString, 1);
+ mr = shims.getMiniSparkCluster(conf, 2, uriString, 1);
} else if (clusterType == MiniClusterType.mr) {
- mr = shims.getMiniMrCluster(conf, 4, uriString, 1);
+ mr = shims.getMiniMrCluster(conf, 2, uriString, 1);
}
}
diff --git pom.xml pom.xml
index 98d2dc2..19a200a 100644
--- pom.xml
+++ pom.xml
@@ -90,6 +90,7 @@
1.0b3
3.3.0-release
+ -Xmx1024m
1.7
2.3
2.12.1
@@ -1001,7 +1002,7 @@
true
false
false
- -Xmx2048m -XX:MaxPermSize=512m
+ ${maven.test.jvm.args}
${test.tmp.dir}/conf
${basedir}/${hive.path.to.root}/conf
diff --git ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
index 23d151c..98d4e6f 100644
--- ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
+++ ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
@@ -105,7 +105,7 @@ Stage-0
<-Map 11 [CONTAINS]
Reduce Output Operator [RS_36]
PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=262/331 width=178)
+ Group By Operator [GBY_35] (rows=262/334 width=178)
Output:["_col0","_col1"],keys:_col1, _col0
Select Operator [SEL_28] (rows=25/25 width=175)
Output:["_col0","_col1"]
@@ -116,7 +116,7 @@ Stage-0
<-Map 16 [CONTAINS]
Reduce Output Operator [RS_36]
PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=262/331 width=178)
+ Group By Operator [GBY_35] (rows=262/334 width=178)
Output:["_col0","_col1"],keys:_col1, _col0
Select Operator [SEL_31] (rows=500/500 width=178)
Output:["_col0","_col1"]
@@ -917,7 +917,7 @@ Stage-0
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_23]
- Group By Operator [GBY_22] (rows=1/1 width=8)
+ Group By Operator [GBY_22] (rows=1/909 width=8)
Output:["_col0"],aggregations:["count()"]
Merge Join Operator [MERGEJOIN_39] (rows=558/1646 width=18)
Conds:Union 2._col0=RS_19._col0(Inner)
diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index ff760c8..ef6b8ae 100644
--- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -293,6 +293,12 @@ public MiniMrShim(Configuration conf, int numberOfTaskTrackers,
JobConf jConf = new JobConf(conf);
jConf.set("yarn.scheduler.capacity.root.queues", "default");
jConf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+ jConf.setInt(MRJobConfig.MAP_MEMORY_MB, 128);
+ jConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128);
+ jConf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128);
+ jConf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512);
+ jConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+ jConf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512);
mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf);
}
@@ -320,6 +326,9 @@ public void setupConfiguration(Configuration conf) {
for (Map.Entry pair: jConf) {
conf.set(pair.getKey(), pair.getValue());
}
+ conf.setInt(MRJobConfig.MAP_MEMORY_MB, 128);
+ conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128);
+ conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128);
}
}
@@ -385,6 +394,16 @@ public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException {
mr = new MiniTezCluster("hive", numberOfTaskTrackers);
+ conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 1024);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 256);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
+ // Overrides values from the hive/tez-site.
+ conf.setInt("hive.tez.container.size", 256);
+ conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 256);
+ conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 256);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 60);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 30);
+ conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f);
conf.set("fs.defaultFS", nameNode);
conf.set("tez.am.log.level", "DEBUG");
conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");
@@ -416,6 +435,13 @@ public void setupConfiguration(Configuration conf) {
for (Map.Entry pair: config) {
conf.set(pair.getKey(), pair.getValue());
}
+ // Overrides values from the hive/tez-site.
+ conf.setInt("hive.tez.container.size", 256);
+ conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 256);
+ conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 256);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 60);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 30);
+ conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f);
}
}
diff --git testutils/ptest2/src/main/resources/batch-exec.vm testutils/ptest2/src/main/resources/batch-exec.vm
index 00487ce..2d16ca3 100644
--- testutils/ptest2/src/main/resources/batch-exec.vm
+++ testutils/ptest2/src/main/resources/batch-exec.vm
@@ -35,7 +35,7 @@ then
export PATH=$JAVA_HOME/bin/:$PATH
fi
export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${antEnvOpts}"
-export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}"
+export MAVEN_OPTS="-Xmx256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}"
export HADOOP_ROOT_LOGGER=INFO,console
export HADOOP_OPTS="-Dhive.log.dir=$logDir -Dhive.query.id=hadoop -Djava.io.tmpdir=$logDir/tmp"
cd $localDir/$instanceName/${repositoryName}-source || exit 1
diff --git testutils/ptest2/src/main/resources/source-prep.vm testutils/ptest2/src/main/resources/source-prep.vm
index 0fc22be..7ad5024 100644
--- testutils/ptest2/src/main/resources/source-prep.vm
+++ testutils/ptest2/src/main/resources/source-prep.vm
@@ -23,7 +23,7 @@ then
export PATH=$JAVA_HOME/bin/:$PATH
fi
export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m ${antEnvOpts}"
-export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m ${mavenEnvOpts}"
+export MAVEN_OPTS="-Xmx1g ${mavenEnvOpts}"
cd $workingDir/
(
if [[ "$clearLibraryCache" == "true" ]]