diff --git data/conf/llap/tez-site.xml data/conf/llap/tez-site.xml index 6f1b9d2..62f97c5 100644 --- data/conf/llap/tez-site.xml +++ data/conf/llap/tez-site.xml @@ -5,6 +5,11 @@ org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled + + tez.am.resource.memory.mb + 128 + + tez.am.task.max.failed.attempts diff --git data/conf/perf-reg/tez-site.xml data/conf/perf-reg/tez-site.xml index 940f390..c575544 100644 --- data/conf/perf-reg/tez-site.xml +++ data/conf/perf-reg/tez-site.xml @@ -1,5 +1,9 @@ + tez.am.resource.memory.mb + 128 + + tez.am.dag.scheduler.class org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled diff --git data/conf/tez/tez-site.xml data/conf/tez/tez-site.xml index 940f390..c575544 100644 --- data/conf/tez/tez-site.xml +++ data/conf/tez/tez-site.xml @@ -1,5 +1,9 @@ + tez.am.resource.memory.mb + 128 + + tez.am.dag.scheduler.class org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled diff --git itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 176761f..3f99579 100644 --- itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -208,7 +208,8 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); + // TODO: This should be making use of confDir to load configs setup for Tez, etc. + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString); break; case LLAP: if (usePortsFromConf) { @@ -216,10 +217,10 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM } llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null); - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString); break; case MR: - mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1); + mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1); break; default: throw new IllegalArgumentException("Unsupported cluster type " + mr); diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index e49ecd9..15e6e93 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -628,12 +628,10 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); } - int numTrackers; + int numTrackers = 2; if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); - numTrackers = 2; } else { - numTrackers = 4; } if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) { mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local); @@ -641,9 +639,9 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws mr = shims.getMiniTezCluster(conf, numTrackers, uriString); } } else if (clusterType == MiniClusterType.miniSparkOnYarn) { - mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); + mr = shims.getMiniSparkCluster(conf, 2, uriString, 1); } else if (clusterType == MiniClusterType.mr) { - mr = shims.getMiniMrCluster(conf, 4, uriString, 1); + mr = shims.getMiniMrCluster(conf, 2, uriString, 1); } } diff --git pom.xml pom.xml index d8276f6..64f53df 100644 --- pom.xml +++ pom.xml @@ -90,6 +90,7 @@ 1.0b3 3.3.0-release + -Xmx1024m 1.7 2.3 2.12.1 @@ -1000,7 +1001,7 @@ true false false - -Xmx2048m -XX:MaxPermSize=512m + ${maven.test.jvm.args} ${test.tmp.dir}/conf ${basedir}/${hive.path.to.root}/conf diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 341b1e5..df15189 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -298,6 +298,12 @@ public MiniMrShim(Configuration conf, int numberOfTaskTrackers, JobConf jConf = new JobConf(conf); jConf.set("yarn.scheduler.capacity.root.queues", "default"); jConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); + jConf.setInt(MRJobConfig.MAP_MEMORY_MB, 128); + jConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128); + jConf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128); + jConf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); + jConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + jConf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf); } @@ -325,6 +331,9 @@ public void setupConfiguration(Configuration conf) { for (Map.Entry pair: jConf) { conf.set(pair.getKey(), pair.getValue()); } + conf.setInt(MRJobConfig.MAP_MEMORY_MB, 128); + conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128); + conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128); } } @@ -390,6 +399,14 @@ public MiniMrShim getMiniTezCluster(Configuration conf, int numberOfTaskTrackers public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); + conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); + // Overrides values from the hive/tez-site. + conf.setInt("hive.tez.container.size", 128); + conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 50); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); @@ -421,6 +438,11 @@ public void setupConfiguration(Configuration conf) { for (Map.Entry pair: config) { conf.set(pair.getKey(), pair.getValue()); } + // Overrides values from the hive/tez-site. + conf.setInt("hive.tez.container.size", 128); + conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 50); } } @@ -460,6 +482,9 @@ public MiniSparkShim(Configuration conf, int numberOfTaskTrackers, conf.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); // disable resource monitoring, although it should be off by default conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false); + conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 4096); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 4096); configureImpersonation(conf); mr.init(conf); mr.start(); diff --git testutils/ptest2/src/main/resources/batch-exec.vm testutils/ptest2/src/main/resources/batch-exec.vm index 2cc56ea..da82a6d 100644 --- testutils/ptest2/src/main/resources/batch-exec.vm +++ testutils/ptest2/src/main/resources/batch-exec.vm @@ -35,7 +35,7 @@ then export PATH=$JAVA_HOME/bin/:$PATH fi export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${antEnvOpts}" -export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}" +export MAVEN_OPTS="-Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}" export HADOOP_ROOT_LOGGER=INFO,console export HADOOP_OPTS="-Dhive.log.dir=$logDir -Dhive.query.id=hadoop -Djava.io.tmpdir=$logDir/tmp" cd $localDir/$instanceName/${repositoryName}-source || exit 1 diff --git testutils/ptest2/src/main/resources/source-prep.vm testutils/ptest2/src/main/resources/source-prep.vm index 67e6a95..3cab8aa 100644 --- testutils/ptest2/src/main/resources/source-prep.vm +++ testutils/ptest2/src/main/resources/source-prep.vm @@ -23,7 +23,7 @@ then export PATH=$JAVA_HOME/bin/:$PATH fi export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m ${antEnvOpts}" -export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m ${mavenEnvOpts}" +export MAVEN_OPTS="-Xmx1g ${mavenEnvOpts}" cd $workingDir/ ( if [[ "$clearLibraryCache" == "true" ]]