diff --git data/conf/llap/tez-site.xml data/conf/llap/tez-site.xml index 6f1b9d2..62f97c5 100644 --- data/conf/llap/tez-site.xml +++ data/conf/llap/tez-site.xml @@ -5,6 +5,11 @@ org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled + + tez.am.resource.memory.mb + 128 + + tez.am.task.max.failed.attempts diff --git data/conf/perf-reg/tez-site.xml data/conf/perf-reg/tez-site.xml index 940f390..c575544 100644 --- data/conf/perf-reg/tez-site.xml +++ data/conf/perf-reg/tez-site.xml @@ -1,5 +1,9 @@ + tez.am.resource.memory.mb + 128 + + tez.am.dag.scheduler.class org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled diff --git data/conf/tez/tez-site.xml data/conf/tez/tez-site.xml index 940f390..c575544 100644 --- data/conf/tez/tez-site.xml +++ data/conf/tez/tez-site.xml @@ -1,5 +1,9 @@ + tez.am.resource.memory.mb + 128 + + tez.am.dag.scheduler.class org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index f02f0e3..d06c869 100644 --- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -223,7 +223,8 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); + // TODO: This should be making use of confDir to load configs setup for Tez, etc. + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString); break; case LLAP: if (usePortsFromConf) { @@ -231,10 +232,10 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM } llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null); - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString); break; case MR: - mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1); + mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1); break; default: throw new IllegalArgumentException("Unsupported cluster type " + mr); diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 0525335..06c1a78 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -628,12 +628,10 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); } - int numTrackers; + int numTrackers = 2; if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); - numTrackers = 2; } else { - numTrackers = 4; } if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) { mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local); @@ -641,9 +639,9 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws mr = shims.getMiniTezCluster(conf, numTrackers, uriString); } } else if (clusterType == MiniClusterType.miniSparkOnYarn) { - mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); + mr = shims.getMiniSparkCluster(conf, 2, uriString, 1); } else if (clusterType == MiniClusterType.mr) { - mr = shims.getMiniMrCluster(conf, 4, uriString, 1); + mr = shims.getMiniMrCluster(conf, 2, uriString, 1); } } diff --git pom.xml pom.xml index 6dad4cd..9659db3 100644 --- pom.xml +++ pom.xml @@ -91,6 +91,7 @@ 1.0b3 3.3.0-release + -Xmx1024m 1.7 2.3 2.12.1 @@ -1003,7 +1004,7 @@ true false false - -Xmx2048m -XX:MaxPermSize=512m + ${maven.test.jvm.args} ${test.conf.dir} ${basedir}/${hive.path.to.root}/conf diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index ff760c8..e56463b 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -293,6 +293,12 @@ public MiniMrShim(Configuration conf, int numberOfTaskTrackers, JobConf jConf = new JobConf(conf); jConf.set("yarn.scheduler.capacity.root.queues", "default"); jConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); + jConf.setInt(MRJobConfig.MAP_MEMORY_MB, 128); + jConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128); + jConf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128); + jConf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); + jConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + jConf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf); } @@ -320,6 +326,9 @@ public void setupConfiguration(Configuration conf) { for (Map.Entry pair: jConf) { conf.set(pair.getKey(), pair.getValue()); } + conf.setInt(MRJobConfig.MAP_MEMORY_MB, 128); + conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128); + conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128); } } @@ -385,6 +394,16 @@ public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); + conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); + // Overrides values from the hive/tez-site. + conf.setInt("hive.tez.container.size", 128); + conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); + conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); @@ -416,6 +435,13 @@ public void setupConfiguration(Configuration conf) { for (Map.Entry pair: config) { conf.set(pair.getKey(), pair.getValue()); } + // Overrides values from the hive/tez-site. + conf.setInt("hive.tez.container.size", 128); + conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); + conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); } } diff --git testutils/ptest2/src/main/resources/batch-exec.vm testutils/ptest2/src/main/resources/batch-exec.vm index 00487ce..2d16ca3 100644 --- testutils/ptest2/src/main/resources/batch-exec.vm +++ testutils/ptest2/src/main/resources/batch-exec.vm @@ -35,7 +35,7 @@ then export PATH=$JAVA_HOME/bin/:$PATH fi export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${antEnvOpts}" -export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}" +export MAVEN_OPTS="-Xmx256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}" export HADOOP_ROOT_LOGGER=INFO,console export HADOOP_OPTS="-Dhive.log.dir=$logDir -Dhive.query.id=hadoop -Djava.io.tmpdir=$logDir/tmp" cd $localDir/$instanceName/${repositoryName}-source || exit 1 diff --git testutils/ptest2/src/main/resources/source-prep.vm testutils/ptest2/src/main/resources/source-prep.vm index 0fc22be..7ad5024 100644 --- testutils/ptest2/src/main/resources/source-prep.vm +++ testutils/ptest2/src/main/resources/source-prep.vm @@ -23,7 +23,7 @@ then export PATH=$JAVA_HOME/bin/:$PATH fi export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m ${antEnvOpts}" -export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m ${mavenEnvOpts}" +export MAVEN_OPTS="-Xmx1g ${mavenEnvOpts}" cd $workingDir/ ( if [[ "$clearLibraryCache" == "true" ]]