diff --git data/conf/llap/hive-site.xml data/conf/llap/hive-site.xml index d93f8b7..f4a9c74 100644 --- data/conf/llap/hive-site.xml +++ data/conf/llap/hive-site.xml @@ -40,7 +40,7 @@ hive.tez.container.size - 128 + 256 diff --git data/conf/llap/tez-site.xml data/conf/llap/tez-site.xml index 6f1b9d2..8e72493 100644 --- data/conf/llap/tez-site.xml +++ data/conf/llap/tez-site.xml @@ -5,6 +5,11 @@ org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled + + tez.am.resource.memory.mb + 256 + + tez.am.task.max.failed.attempts diff --git data/conf/perf-reg/tez-site.xml data/conf/perf-reg/tez-site.xml index 940f390..c575544 100644 --- data/conf/perf-reg/tez-site.xml +++ data/conf/perf-reg/tez-site.xml @@ -1,5 +1,9 @@ + tez.am.resource.memory.mb + 128 + + tez.am.dag.scheduler.class org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled diff --git data/conf/tez/hive-site.xml data/conf/tez/hive-site.xml index dbff10c..4f0fc4c 100644 --- data/conf/tez/hive-site.xml +++ data/conf/tez/hive-site.xml @@ -40,7 +40,7 @@ hive.tez.container.size - 128 + 256 diff --git data/conf/tez/tez-site.xml data/conf/tez/tez-site.xml index 940f390..f0d2ffb 100644 --- data/conf/tez/tez-site.xml +++ data/conf/tez/tez-site.xml @@ -1,5 +1,9 @@ + tez.am.resource.memory.mb + 256 + + tez.am.dag.scheduler.class org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index ddc2690..2dbd04f 100644 --- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -209,7 +209,8 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); + // TODO: This should be making use of confDir to load configs setup for Tez, etc. + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString); break; case LLAP: if (usePortsFromConf) { @@ -217,10 +218,10 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM } llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null); - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString); break; case MR: - mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1); + mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1); break; default: throw new IllegalArgumentException("Unsupported cluster type " + mr); diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index ff640d3..230fdaa 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -628,12 +628,10 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); } - int numTrackers; + int numTrackers = 2; if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); - numTrackers = 2; } else { - numTrackers = 4; } if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) { mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local); @@ -641,9 +639,9 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws mr = shims.getMiniTezCluster(conf, numTrackers, uriString); } } else if (clusterType == MiniClusterType.miniSparkOnYarn) { - mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); + mr = shims.getMiniSparkCluster(conf, 2, uriString, 1); } else if (clusterType == MiniClusterType.mr) { - mr = shims.getMiniMrCluster(conf, 4, uriString, 1); + mr = shims.getMiniMrCluster(conf, 2, uriString, 1); } } diff --git pom.xml pom.xml index 98d2dc2..19a200a 100644 --- pom.xml +++ pom.xml @@ -90,6 +90,7 @@ 1.0b3 3.3.0-release + -Xmx1024m 1.7 2.3 2.12.1 @@ -1001,7 +1002,7 @@ true false false - -Xmx2048m -XX:MaxPermSize=512m + ${maven.test.jvm.args} ${test.tmp.dir}/conf ${basedir}/${hive.path.to.root}/conf diff --git ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out index 23d151c..98d4e6f 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out @@ -105,7 +105,7 @@ Stage-0 <-Map 11 [CONTAINS] Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262/331 width=178) + Group By Operator [GBY_35] (rows=262/334 width=178) Output:["_col0","_col1"],keys:_col1, _col0 Select Operator [SEL_28] (rows=25/25 width=175) Output:["_col0","_col1"] @@ -116,7 +116,7 @@ Stage-0 <-Map 16 [CONTAINS] Reduce Output Operator [RS_36] PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=262/331 width=178) + Group By Operator [GBY_35] (rows=262/334 width=178) Output:["_col0","_col1"],keys:_col1, _col0 Select Operator [SEL_31] (rows=500/500 width=178) Output:["_col0","_col1"] @@ -917,7 +917,7 @@ Stage-0 Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1/1 width=8) + Group By Operator [GBY_22] (rows=1/909 width=8) Output:["_col0"],aggregations:["count()"] Merge Join Operator [MERGEJOIN_39] (rows=558/1646 width=18) Conds:Union 2._col0=RS_19._col0(Inner) diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index ff760c8..ef6b8ae 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -293,6 +293,12 @@ public MiniMrShim(Configuration conf, int numberOfTaskTrackers, JobConf jConf = new JobConf(conf); jConf.set("yarn.scheduler.capacity.root.queues", "default"); jConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); + jConf.setInt(MRJobConfig.MAP_MEMORY_MB, 128); + jConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128); + jConf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128); + jConf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); + jConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + jConf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf); } @@ -320,6 +326,9 @@ public void setupConfiguration(Configuration conf) { for (Map.Entry pair: jConf) { conf.set(pair.getKey(), pair.getValue()); } + conf.setInt(MRJobConfig.MAP_MEMORY_MB, 128); + conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128); + conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128); } } @@ -385,6 +394,16 @@ public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); + conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 256); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); + // Overrides values from the hive/tez-site. + conf.setInt("hive.tez.container.size", 256); + conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 256); + conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 256); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 60); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 30); + conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); @@ -416,6 +435,13 @@ public void setupConfiguration(Configuration conf) { for (Map.Entry pair: config) { conf.set(pair.getKey(), pair.getValue()); } + // Overrides values from the hive/tez-site. + conf.setInt("hive.tez.container.size", 256); + conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 256); + conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 256); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 60); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 30); + conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); } } diff --git testutils/ptest2/src/main/resources/batch-exec.vm testutils/ptest2/src/main/resources/batch-exec.vm index 00487ce..2d16ca3 100644 --- testutils/ptest2/src/main/resources/batch-exec.vm +++ testutils/ptest2/src/main/resources/batch-exec.vm @@ -35,7 +35,7 @@ then export PATH=$JAVA_HOME/bin/:$PATH fi export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${antEnvOpts}" -export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}" +export MAVEN_OPTS="-Xmx256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}" export HADOOP_ROOT_LOGGER=INFO,console export HADOOP_OPTS="-Dhive.log.dir=$logDir -Dhive.query.id=hadoop -Djava.io.tmpdir=$logDir/tmp" cd $localDir/$instanceName/${repositoryName}-source || exit 1 diff --git testutils/ptest2/src/main/resources/source-prep.vm testutils/ptest2/src/main/resources/source-prep.vm index 0fc22be..7ad5024 100644 --- testutils/ptest2/src/main/resources/source-prep.vm +++ testutils/ptest2/src/main/resources/source-prep.vm @@ -23,7 +23,7 @@ then export PATH=$JAVA_HOME/bin/:$PATH fi export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m ${antEnvOpts}" -export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m ${mavenEnvOpts}" +export MAVEN_OPTS="-Xmx1g ${mavenEnvOpts}" cd $workingDir/ ( if [[ "$clearLibraryCache" == "true" ]]