diff --git hcatalog/webhcat/svr/src/main/config/override-container-log4j.properties hcatalog/webhcat/svr/src/main/config/override-container-log4j.properties deleted file mode 100644 index f6b740f..0000000 --- hcatalog/webhcat/svr/src/main/config/override-container-log4j.properties +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# -# This log4j config overrides hadoop-yarn-server-nodemanager-2.1.0-beta.jar/container-log4j.properties. -#In Hadoop 2, (by default) the log information about M/R job progress is not sent to stderr, -#which is where LaunchMapper expects it. Thus WebHCat is unable to report the -#percentComplete attribute in job status. There is something broken in YARN that doesn't allow -#its log4j properties to be overridden. Thus for now (10/07/2013) we resort to overriding it -#using this file, where log4j.rootLogger specify additional 'console' appender. This file is made -#available through DistributedCache. See TrivialExecService and TempletonControllerJob for more -#info. - -hadoop.root.logger=INFO,CLA - -# Define the root logger to the system property "hadoop.root.logger". -log4j.rootLogger=${hadoop.root.logger}, console, EventCounter - -# Logging Threshold -log4j.threshold=ALL - -# -# ContainerLog Appender -# - -#Default values -yarn.app.container.log.dir=null -yarn.app.container.log.filesize=100 - -log4j.appender.CLA=org.apache.hadoop.yarn.ContainerLogAppender -log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir} -log4j.appender.CLA.totalLogFileSize=${yarn.app.container.log.filesize} - -log4j.appender.CLA.layout=org.apache.log4j.PatternLayout -log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n - -# -# Event Counter Appender -# Sends counts of logging messages at different severity levels to Hadoop Metrics. -# -log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter - - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java index 5f6c18c..a6355a6 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java @@ -25,9 +25,6 @@ public static final String JOB_TYPE = "templeton.jobtype"; public static final String JAR_ARGS_NAME = "templeton.args"; public static final String OVERRIDE_CLASSPATH = "templeton.override-classpath"; - public static final String OVERRIDE_CONTAINER_LOG4J_PROPS = "override.containerLog4j"; - //name of file - static final String CONTAINER_LOG4J_PROPS = "override-container-log4j.properties"; public static final String STDOUT_FNAME = "stdout"; public static final String STDERR_FNAME = "stderr"; public static final String EXIT_FNAME = "exit"; diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java index 3c85dd4..e22399c 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java @@ -103,9 +103,7 @@ protected Process startJob(Context context, String user, String overrideClasspat List jarArgsList = new LinkedList(Arrays.asList(jarArgs)); handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary"); handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path"); - boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ? - false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS)); - return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps); + return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env); } /** diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java index 4807e79..88d82c3 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java @@ -74,87 +74,6 @@ @InterfaceAudience.Private public class TempletonControllerJob extends Configured implements Tool, JobSubmissionConstants { private static final Log LOG = LogFactory.getLog(TempletonControllerJob.class); - //file to add to DistributedCache - private static URI overrideLog4jURI = null; - private static boolean overrideContainerLog4jProps; - //Jar cmd submission likely will be affected, Pig likely not - private static final String affectedMsg = "Monitoring of Hadoop jobs submitted through WebHCat " + - "may be affected."; - private static final String TMP_DIR_PROP = "hadoop.tmp.dir"; - - /** - * Copy the file from local file system to tmp dir - */ - private static URI copyLog4JtoFileSystem(final String localFile) throws IOException, - InterruptedException { - UserGroupInformation ugi = UserGroupInformation.getLoginUser(); - return ugi.doAs(new PrivilegedExceptionAction() { - @Override - public URI run() throws IOException { - AppConfig appConfig = Main.getAppConfigInstance(); - String fsTmpDir = appConfig.get(TMP_DIR_PROP); - if(fsTmpDir == null || fsTmpDir.length() <= 0) { - LOG.warn("Could not find 'hadoop.tmp.dir'; " + affectedMsg); - return null; - } - FileSystem fs = FileSystem.get(appConfig); - Path dirPath = new Path(fsTmpDir); - if(!fs.exists(dirPath)) { - LOG.warn(dirPath + " does not exist; " + affectedMsg); - return null; - } - Path dst = fs.makeQualified(new Path(fsTmpDir, CONTAINER_LOG4J_PROPS)); - fs.copyFromLocalFile(new Path(localFile), dst); - //make readable by all users since TempletonControllerJob#run() is run as submitting user - fs.setPermission(dst, new FsPermission((short)0644)); - return dst.toUri(); - } - }); - } - /** - * local file system - * @return - */ - private static String getLog4JPropsLocal() { - return AppConfig.getWebhcatConfDir() + File.separator + CONTAINER_LOG4J_PROPS; - } - static { - //initialize once-per-JVM (i.e. one running WebHCat server) state and log it once since it's - // the same for every job - try { - //safe (thread) publication - // http://docs.oracle.com/javase/specs/jls/se5.0/html/execution.html#12.4.2 - LOG.info("Using Hadoop Version: " + ShimLoader.getMajorVersion()); - overrideContainerLog4jProps = "0.23".equals(ShimLoader.getMajorVersion()); - if(overrideContainerLog4jProps) { - //see detailed note in CONTAINER_LOG4J_PROPS file - LOG.info(AppConfig.WEBHCAT_CONF_DIR + "=" + AppConfig.getWebhcatConfDir()); - File localFile = new File(getLog4JPropsLocal()); - if(localFile.exists()) { - LOG.info("Found " + localFile.getAbsolutePath() + " to use for job submission."); - try { - overrideLog4jURI = copyLog4JtoFileSystem(getLog4JPropsLocal()); - LOG.info("Job submission will use log4j.properties=" + overrideLog4jURI); - } - catch(IOException ex) { - LOG.warn("Will not add " + CONTAINER_LOG4J_PROPS + " to Distributed Cache. " + - "Some fields in job status may be unavailable", ex); - } - } - else { - LOG.warn("Could not find " + localFile.getAbsolutePath() + ". " + affectedMsg); - } - } - } - catch(Throwable t) { - //this intentionally doesn't use TempletonControllerJob.class.getName() to be able to - //log errors which may be due to class loading - String msg = "org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob is not " + - "properly initialized. " + affectedMsg; - LOG.error(msg, t); - } - } - private final boolean secureMetastoreAccess; /** @@ -191,10 +110,6 @@ public int run(String[] args) throws IOException, InterruptedException, ClassNot conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args)); String user = UserGroupInformation.getCurrentUser().getShortUserName(); conf.set("user.name", user); - if(overrideContainerLog4jProps && overrideLog4jURI != null) { - //must be done before Job object is created - conf.set(OVERRIDE_CONTAINER_LOG4J_PROPS, Boolean.TRUE.toString()); - } Job job = new Job(conf); job.setJarByClass(LaunchMapper.class); job.setJobName(TempletonControllerJob.class.getSimpleName()); @@ -202,20 +117,6 @@ public int run(String[] args) throws IOException, InterruptedException, ClassNot job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setInputFormatClass(SingleInputFormat.class); - if(overrideContainerLog4jProps && overrideLog4jURI != null) { - FileSystem fs = FileSystem.get(conf); - if(fs.exists(new Path(overrideLog4jURI))) { - ShimLoader.getHadoopShims().getWebHCatShim(conf, UgiFactory.getUgi(user)).addCacheFile( - overrideLog4jURI, job); - LOG.debug("added " + overrideLog4jURI + " to Dist Cache"); - } - else { - //in case this file was deleted by someone issue a warning but don't try to add to - // DistributedCache as that will throw and fail job submission - LOG.warn("Cannot find " + overrideLog4jURI + " which is created on WebHCat startup/job " + - "submission. " + affectedMsg); - } - } NullOutputFormat of = new NullOutputFormat(); job.setOutputFormatClass(of.getClass()); @@ -238,10 +139,6 @@ public int run(String[] args) throws IOException, InterruptedException, ClassNot LOG.debug("Added metastore delegation token for jobId=" + submittedJobId.toString() + " user=" + user); } - if(overrideContainerLog4jProps && overrideLog4jURI == null) { - //do this here so that log msg has JobID - LOG.warn("Could not override container log4j properties for " + submittedJobId); - } return 0; } private String addHMSToken(Job job, String user) throws IOException, InterruptedException, diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java index a2038ec..32f1ca8 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java @@ -39,10 +39,8 @@ * not available on every node in the cluster (outside webhcat jar) */ final class TrivialExecService { - //with default log4j config, this output ends up in 'syslog' of the LaunchMapper task private static final Log LOG = LogFactory.getLog(TrivialExecService.class); private static volatile TrivialExecService theSingleton; - private static final String HADOOP_CLIENT_OPTS = "HADOOP_CLIENT_OPTS"; /** * Retrieve the singleton. */ @@ -51,31 +49,9 @@ public static synchronized TrivialExecService getInstance() { theSingleton = new TrivialExecService(); return theSingleton; } - /** - * See {@link JobSubmissionConstants#CONTAINER_LOG4J_PROPS} file for details. - */ - private static void hadoop2LogRedirect(ProcessBuilder processBuilder) { - Map env = processBuilder.environment(); - if(!env.containsKey(HADOOP_CLIENT_OPTS)) { - return; - } - String hcopts = env.get(HADOOP_CLIENT_OPTS); - if(!hcopts.contains("log4j.configuration=container-log4j.properties")) { - return; - } - //TempletonControllerJob ensures that this file is in DistributedCache - File log4jProps = new File(JobSubmissionConstants.CONTAINER_LOG4J_PROPS); - hcopts = hcopts.replace("log4j.configuration=container-log4j.properties", - "log4j.configuration=file://" + new Path(log4jProps.getAbsolutePath()).toUri().toString()); - //helps figure out what log4j is doing, but may confuse - //some jobs due to extra output to stdout - //hcopts = hcopts + " -Dlog4j.debug=true"; - env.put(HADOOP_CLIENT_OPTS, hcopts); - } public Process run(List cmd, List removeEnv, - Map environmentVariables, boolean overrideContainerLog4jProps) - throws IOException { - LOG.info("run(cmd, removeEnv, environmentVariables, " + overrideContainerLog4jProps + ")"); + Map environmentVariables) throws IOException { + LOG.info("run(cmd, removeEnv, environmentVariables)"); LOG.info("Starting cmd: " + cmd); ProcessBuilder pb = new ProcessBuilder(cmd); for (String key : removeEnv) { @@ -85,9 +61,6 @@ public Process run(List cmd, List removeEnv, pb.environment().remove(key); } pb.environment().putAll(environmentVariables); - if(overrideContainerLog4jProps) { - hadoop2LogRedirect(pb); - } logDebugInfo("Starting process with env:", pb.environment()); return pb.start(); } diff --git hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTrivialExecService.java hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTrivialExecService.java index b76e69a..a873a96 100644 --- hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTrivialExecService.java +++ hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTrivialExecService.java @@ -38,7 +38,7 @@ public void test() { Process process = TrivialExecService.getInstance() .run(list, new ArrayList(), - new HashMap(),false); + new HashMap()); out = new BufferedReader(new InputStreamReader( process.getInputStream())); err = new BufferedReader(new InputStreamReader( diff --git packaging/src/main/assembly/bin.xml packaging/src/main/assembly/bin.xml index 892b41f..860d5e7 100644 --- packaging/src/main/assembly/bin.xml +++ packaging/src/main/assembly/bin.xml @@ -272,7 +272,6 @@ webhcat-default.xml webhcat-log4j.properties - override-container-log4j.properties hcatalog/etc/webhcat diff --git shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java index 2bc3ca4..638c107 100644 --- shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java +++ shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java @@ -37,12 +37,8 @@ public WebHCatJTShim23(final Configuration conf, final UserGroupInformation ugi) try { jc = ugi.doAs(new PrivilegedExceptionAction() { public JobClient run() throws IOException, InterruptedException { - return ugi.doAs(new PrivilegedExceptionAction() { - public JobClient run() throws IOException { - //create this in doAs() so that it gets a security context based passed in 'ugi' - return new JobClient(conf); - } - }); + //create this in doAs() so that it gets a security context based passed in 'ugi' + return new JobClient(conf); } }); }