diff --git hcatalog/webhcat/svr/src/main/config/webhcat-default.xml hcatalog/webhcat/svr/src/main/config/webhcat-default.xml index 5344a0f..f88b267 100644 --- hcatalog/webhcat/svr/src/main/config/webhcat-default.xml +++ hcatalog/webhcat/svr/src/main/config/webhcat-default.xml @@ -197,6 +197,32 @@ + + templeton.exec.envs HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java index 37ce997..eb80e63 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java @@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.SystemVariables; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.VersionInfo; import org.apache.hive.hcatalog.templeton.tool.JobState; @@ -104,6 +103,8 @@ public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive"; public static final String HIVE_PATH_NAME = "templeton.hive.path"; public static final String MAPPER_MEMORY_MB = "templeton.mapper.memory.mb"; + public static final String MR_AM_MEMORY_MB = "templeton.mr.am.memory.mb"; + /** * see webhcat-default.xml */ @@ -130,6 +131,8 @@ public static final String OVERRIDE_JARS_ENABLED = "templeton.override.enabled"; public static final String TEMPLETON_CONTROLLER_MR_CHILD_OPTS = "templeton.controller.mr.child.opts"; + public static final String TEMPLETON_CONTROLLER_MR_AM_JAVA_OPTS + = "templeton.controller.mr.am.java.opts"; public static final String KERBEROS_SECRET = "templeton.kerberos.secret"; public static final String KERBEROS_PRINCIPAL = "templeton.kerberos.principal"; @@ -148,6 +151,8 @@ = "mapred.map.tasks.speculative.execution"; public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts"; public static final String HADOOP_MAP_MEMORY_MB = "mapreduce.map.memory.mb"; + public static final String HADOOP_MR_AM_JAVA_OPTS = "yarn.app.mapreduce.am.command-opts"; + public static final String HADOOP_MR_AM_MEMORY_MB = "yarn.app.mapreduce.am.resource.mb"; public static final String UNIT_TEST_MODE = "templeton.unit.test.mode"; @@ -313,7 +318,13 @@ private boolean loadOneClasspathConfig(String fname) { public String controllerMRChildOpts() { return get(TEMPLETON_CONTROLLER_MR_CHILD_OPTS); } + public String controllerAMChildOpts() { + return get(TEMPLETON_CONTROLLER_MR_AM_JAVA_OPTS); + } public String mapperMemoryMb() { return get(MAPPER_MEMORY_MB); } + public String amMemoryMb() { + return get(MR_AM_MEMORY_MB); + } /** * @see #HIVE_PROPS_NAME diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java index 53eecfa..92e45cb 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java @@ -18,9 +18,7 @@ */ package org.apache.hive.hcatalog.templeton.tool; -import java.io.File; import java.io.IOException; -import java.net.URI; import java.security.PrivilegedExceptionAction; import java.util.Arrays; @@ -28,13 +26,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobClient; @@ -47,7 +41,6 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Tool; import org.apache.hive.hcatalog.templeton.AppConfig; -import org.apache.hive.hcatalog.templeton.Main; import org.apache.hive.hcatalog.templeton.SecureProxySupport; import org.apache.hive.hcatalog.templeton.UgiFactory; import org.apache.thrift.TException; @@ -114,6 +107,15 @@ public int run(String[] args) throws IOException, InterruptedException, ClassNot if(memoryMb != null && memoryMb.length() != 0) { conf.set(AppConfig.HADOOP_MAP_MEMORY_MB, memoryMb); } + String amMemoryMB = appConf.amMemoryMb(); + if (amMemoryMB != null && !amMemoryMB.isEmpty()) { + conf.set(AppConfig.HADOOP_MR_AM_MEMORY_MB, amMemoryMB); + } + String amJavaOpts = appConf.controllerAMChildOpts(); + if (amJavaOpts != null && !amJavaOpts.isEmpty()) { + conf.set(AppConfig.HADOOP_MR_AM_JAVA_OPTS, amJavaOpts); + } + String user = UserGroupInformation.getCurrentUser().getShortUserName(); conf.set("user.name", user); Job job = new Job(conf);