diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java index 77952be..f001f72 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java @@ -28,12 +28,14 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.StringTokenizer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.SystemVariables; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.VersionInfo; import org.apache.hive.hcatalog.templeton.tool.JobState; @@ -224,14 +226,8 @@ private static void logConfigLoadAttempt(String path) { * support/debugging. Later it may be worth adding a REST call which will return this data. */ private String dumpEnvironent() { - StringBuilder sb = new StringBuilder("WebHCat environment:\n"); - Map env = System.getenv(); - List propKeys = new ArrayList(env.keySet()); - Collections.sort(propKeys); - for(String propKey : propKeys) { - sb.append(propKey).append('=').append(env.get(propKey)).append('\n'); - } - sb.append("Configration properties: \n"); + StringBuilder sb = TempletonUtils.dumpPropMap("========WebHCat System.getenv()========", System.getenv()); + sb.append("START========WebHCat AppConfig.iterator()========: \n"); Iterator> configIter = this.iterator(); List> configVals = new ArrayList>(); while(configIter.hasNext()) { @@ -245,8 +241,19 @@ public int compare(Map.Entry ent, Map.Entry ent2 }); for(Map.Entry entry : configVals) { //use get() to make sure variable substitution works - sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n'); + if(entry.getKey().toLowerCase().contains("path")) { + StringTokenizer st = new StringTokenizer(get(entry.getKey()), File.pathSeparator); + sb.append(entry.getKey()).append("=\n"); + while(st.hasMoreTokens()) { + sb.append(" ").append(st.nextToken()).append(File.pathSeparator).append('\n'); + } + } + else { + sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n'); + } } + sb.append("END========WebHCat AppConfig.iterator()========: \n"); + sb.append(TempletonUtils.dumpPropMap("========WebHCat System.getProperties()========", System.getProperties())); return sb.toString(); } public void startCleanup() { diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java index 902d786..fb4b8e7 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java @@ -18,6 +18,7 @@ */ package org.apache.hive.hcatalog.templeton.tool; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -34,6 +35,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.StringTokenizer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -408,4 +411,34 @@ public static String findContainingJar(Class clazz, String fileNamePattern) { } return null; } + public static StringBuilder dumpPropMap(String header, Properties props) { + Map map = new HashMap(); + for(Map.Entry ent : props.entrySet()) { + map.put(ent.getKey().toString(), ent.getValue() == null ? null : ent.getValue().toString()); + } + return dumpPropMap(header, map); + } + public static StringBuilder dumpPropMap(String header, Map map) { + StringBuilder sb = new StringBuilder("START").append(header).append(":\n"); + List propKeys = new ArrayList(map.keySet()); + Collections.sort(propKeys); + for(String propKey : propKeys) { + if(propKey.toLowerCase().contains("path")) { + StringTokenizer st = new StringTokenizer(map.get(propKey), File.pathSeparator); + if(st.countTokens() > 1) { + sb.append(propKey).append("=\n"); + while (st.hasMoreTokens()) { + sb.append(" ").append(st.nextToken()).append(File.pathSeparator).append('\n'); + } + } + else { + sb.append(propKey).append('=').append(map.get(propKey)).append('\n'); + } + } + else { + sb.append(propKey).append('=').append(map.get(propKey)).append('\n'); + } + } + return sb.append("END").append(header).append('\n'); + } } diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java index 32f1ca8..892d79e 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java @@ -20,7 +20,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; import java.io.File; import java.io.IOException; @@ -28,6 +27,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.StringTokenizer; /** @@ -61,16 +61,47 @@ public Process run(List cmd, List removeEnv, pb.environment().remove(key); } pb.environment().putAll(environmentVariables); - logDebugInfo("Starting process with env:", pb.environment()); + logDebugInfo("========Starting process with env:========", pb.environment()); + printContentsOfDir("."); return pb.start(); } private static void logDebugInfo(String msg, Map props) { - LOG.info(msg); - List keys = new ArrayList(); - keys.addAll(props.keySet()); - Collections.sort(keys); - for(String key : keys) { - LOG.info(key + "=" + props.get(key)); - } + StringBuilder sb = TempletonUtils.dumpPropMap(msg, props); + LOG.info(sb.toString()); + String sqoopHome = props.get("SQOOP_HOME"); + if(TempletonUtils.isset(sqoopHome)) { + //this is helpful when Sqoop is installed on each node in the cluster to make sure + //relevant jars (JDBC in particular) are present on the node running the command + printContentsOfDir(sqoopHome + File.separator+ "lib"); + } + } + /** + * Print files and directories in current directory. Will list files in the sub-directory (only 1 level deep) + * time honored tradition in WebHCat of borrowing from Oozie + */ + private static void printContentsOfDir(String dir) { + File folder = new File(dir); + StringBuilder sb = new StringBuilder("Files in '").append(dir).append("' dir:").append(folder.getAbsolutePath()).append('\n'); + + File[] listOfFiles = folder.listFiles(); + for (File fileName : listOfFiles) { + if (fileName.isFile()) { + sb.append("File: ").append(fileName.getName()).append('\n'); + } + else if (fileName.isDirectory()) { + sb.append("Dir: ").append(fileName.getName()).append('\n'); + File subDir = new File(fileName.getName()); + File[] moreFiles = subDir.listFiles(); + for (File subFileName : moreFiles) { + if (subFileName.isFile()) { + sb.append("--File: ").append(subFileName.getName()).append('\n'); + } + else if (subFileName.isDirectory()) { + sb.append("--Dir: ").append(subFileName.getName()).append('\n'); + } + } + } + } + LOG.info(sb.toString()); } }