diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 1672453..c276092 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -379,6 +379,9 @@
// whether session is running in silent mode or not
HIVESESSIONSILENT("hive.session.silent", false),
+ // Whether to enable history for this session
+ HIVE_SESSION_HISTORY_ENABLED("hive.session.history.enabled", false),
+
// query being executed (multiple per session)
HIVEQUERYSTRING("hive.query.string", ""),
diff --git a/conf/hive-default.xml.template b/conf/hive-default.xml.template
index 3a7d1dc..1f38bbb 100644
--- a/conf/hive-default.xml.template
+++ b/conf/hive-default.xml.template
@@ -441,6 +441,12 @@
+ hive.session.history.enabled
+ false
+ Whether to log hive query, query plan, runtime statistics etc
+
+
+
hive.map.aggr.hash.min.reduction
0.5
Hash aggregation will be turned off if the ratio between hash
diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml
index 544ba35..407c8f6 100644
--- a/data/conf/hive-site.xml
+++ b/data/conf/hive-site.xml
@@ -53,6 +53,12 @@
+ hive.session.history.enabled
+ true
+ Whether to log hive query, query plan, runtime statistics etc
+
+
+
javax.jdo.option.ConnectionURL
jdbc:derby:;databaseName=../build/test/junit_metastore_db;create=true
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java b/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
index e1c1ae3..3986f89 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
@@ -18,55 +18,21 @@
package org.apache.hadoop.hive.ql.history;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
-import java.util.Random;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.Counters.Counter;
-import org.apache.hadoop.mapred.Counters.Group;
/**
- * HiveHistory.
- *
+ * HiveHistory. Logs information such as query, query plan, runtime statistics
+ * for into a file.
+ * Each session uses a new object, which creates a new file.
*/
-public class HiveHistory {
-
- PrintWriter histStream; // History File stream
-
- String histFileName; // History file name
-
- private static final Log LOG = LogFactory.getLog("hive.ql.exec.HiveHistory");
-
- private LogHelper console;
-
- private Map idToTableMap = null;
-
- // Job Hash Map
- private final HashMap queryInfoMap = new HashMap();
-
- // Task Hash Map
- private final HashMap taskInfoMap = new HashMap();
-
- private static final String DELIMITER = " ";
+public interface HiveHistory {
/**
* RecordTypes.
@@ -105,18 +71,6 @@
ROWS_INSERTED
};
- private static final String KEY = "(\\w+)";
- private static final String VALUE = "[[^\"]?]+"; // anything but a " in ""
- private static final String ROW_COUNT_PATTERN = "TABLE_ID_(\\d+)_ROWCOUNT";
-
- private static final Pattern pattern = Pattern.compile(KEY + "=" + "\""
- + VALUE + "\"");
-
- private static final Pattern rowCountPattern = Pattern.compile(ROW_COUNT_PATTERN);
-
- // temp buffer for parsed dataa
- private static Map parseBuffer = new HashMap();
-
/**
* Listner interface Parser will call handle function for each record type.
*/
@@ -126,63 +80,6 @@
}
/**
- * Parses history file and calls call back functions.
- *
- * @param path
- * @param l
- * @throws IOException
- */
- public static void parseHiveHistory(String path, Listener l) throws IOException {
- FileInputStream fi = new FileInputStream(path);
- BufferedReader reader = new BufferedReader(new InputStreamReader(fi));
- try {
- String line = null;
- StringBuilder buf = new StringBuilder();
- while ((line = reader.readLine()) != null) {
- buf.append(line);
- // if it does not end with " then it is line continuation
- if (!line.trim().endsWith("\"")) {
- continue;
- }
- parseLine(buf.toString(), l);
- buf = new StringBuilder();
- }
- } finally {
- try {
- reader.close();
- } catch (IOException ex) {
- }
- }
- }
-
- /**
- * Parse a single line of history.
- *
- * @param line
- * @param l
- * @throws IOException
- */
- private static void parseLine(String line, Listener l) throws IOException {
- // extract the record type
- int idx = line.indexOf(' ');
- String recType = line.substring(0, idx);
- String data = line.substring(idx + 1, line.length());
-
- Matcher matcher = pattern.matcher(data);
-
- while (matcher.find()) {
- String tuple = matcher.group(0);
- String[] parts = tuple.split("=");
-
- parseBuffer.put(parts[0], parts[1].substring(1, parts[1].length() - 1));
- }
-
- l.handle(RecordTypes.valueOf(recType), parseBuffer);
-
- parseBuffer.clear();
- }
-
- /**
* Info.
*
*/
@@ -217,106 +114,14 @@ private static void parseLine(String line, Listener l) throws IOException {
};
/**
- * Construct HiveHistory object an open history log file.
- *
- * @param ss
- */
- public HiveHistory(SessionState ss) {
-
- try {
- console = new LogHelper(LOG);
- String conf_file_loc = ss.getConf().getVar(
- HiveConf.ConfVars.HIVEHISTORYFILELOC);
- if ((conf_file_loc == null) || conf_file_loc.length() == 0) {
- console.printError("No history file location given");
- return;
- }
-
- // Create directory
- File f = new File(conf_file_loc);
- if (!f.exists()) {
- if (!f.mkdirs()) {
- console.printError("Unable to create log directory " + conf_file_loc);
- return;
- }
- }
- Random randGen = new Random();
- do {
- histFileName = conf_file_loc + "/hive_job_log_" + ss.getSessionId() + "_"
- + Math.abs(randGen.nextInt()) + ".txt";
- } while (new File(histFileName).exists());
- console.printInfo("Hive history file=" + histFileName);
- histStream = new PrintWriter(histFileName);
-
- HashMap hm = new HashMap();
- hm.put(Keys.SESSION_ID.name(), ss.getSessionId());
- log(RecordTypes.SessionStart, hm);
- } catch (FileNotFoundException e) {
- console.printError("FAILED: Failed to open Query Log : " + histFileName
- + " " + e.getMessage(), "\n"
- + org.apache.hadoop.util.StringUtils.stringifyException(e));
- }
-
- }
-
- /**
* @return historyFileName
*/
- public String getHistFileName() {
- return histFileName;
- }
-
- /**
- * Write the a history record to history file.
- *
- * @param rt
- * @param keyValMap
- */
- void log(RecordTypes rt, Map keyValMap) {
-
- if (histStream == null) {
- return;
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append(rt.name());
-
- for (Map.Entry ent : keyValMap.entrySet()) {
-
- sb.append(DELIMITER);
- String key = ent.getKey();
- String val = ent.getValue();
- if(val != null) {
- val = val.replace('\n', ' ');
- }
- sb.append(key + "=\"" + val + "\"");
-
- }
- sb.append(DELIMITER);
- sb.append(Keys.TIME.name() + "=\"" + System.currentTimeMillis() + "\"");
- histStream.println(sb);
- histStream.flush();
-
- }
+ public String getHistFileName();
/**
* Called at the start of job Driver.execute().
*/
- public void startQuery(String cmd, String id) {
- SessionState ss = SessionState.get();
- if (ss == null) {
- return;
- }
- QueryInfo ji = new QueryInfo();
-
- ji.hm.put(Keys.QUERY_ID.name(), id);
- ji.hm.put(Keys.QUERY_STRING.name(), cmd);
-
- queryInfoMap.put(id, ji);
-
- log(RecordTypes.QueryStart, ji.hm);
-
- }
+ public void startQuery(String cmd, String id);
/**
* Used to set job status and other attributes of a job.
@@ -325,13 +130,7 @@ public void startQuery(String cmd, String id) {
* @param propName
* @param propValue
*/
- public void setQueryProperty(String queryId, Keys propName, String propValue) {
- QueryInfo ji = queryInfoMap.get(queryId);
- if (ji == null) {
- return;
- }
- ji.hm.put(propName.name(), propValue);
- }
+ public void setQueryProperty(String queryId, Keys propName, String propValue);
/**
* Used to set task properties.
@@ -341,14 +140,7 @@ public void setQueryProperty(String queryId, Keys propName, String propValue) {
* @param propValue
*/
public void setTaskProperty(String queryId, String taskId, Keys propName,
- String propValue) {
- String id = queryId + ":" + taskId;
- TaskInfo ti = taskInfoMap.get(id);
- if (ti == null) {
- return;
- }
- ti.hm.put(propName.name(), propValue);
- }
+ String propValue);
/**
* Serialize the task counters and set as a task property.
@@ -357,80 +149,16 @@ public void setTaskProperty(String queryId, String taskId, Keys propName,
* @param taskId
* @param ctrs
*/
- public void setTaskCounters(String queryId, String taskId, Counters ctrs) {
- String id = queryId + ":" + taskId;
- QueryInfo ji = queryInfoMap.get(queryId);
- StringBuilder sb1 = new StringBuilder("");
- TaskInfo ti = taskInfoMap.get(id);
- if ((ti == null) || (ctrs == null)) {
- return;
- }
- StringBuilder sb = new StringBuilder("");
- try {
-
- boolean first = true;
- for (Group group : ctrs) {
- for (Counter counter : group) {
- if (first) {
- first = false;
- } else {
- sb.append(',');
- }
- sb.append(group.getDisplayName());
- sb.append('.');
- sb.append(counter.getDisplayName());
- sb.append(':');
- sb.append(counter.getCounter());
- String tab = getRowCountTableName(counter.getDisplayName());
- if (tab != null) {
- if (sb1.length() > 0) {
- sb1.append(",");
- }
- sb1.append(tab);
- sb1.append('~');
- sb1.append(counter.getCounter());
- ji.rowCountMap.put(tab, counter.getCounter());
-
- }
- }
- }
+ public void setTaskCounters(String queryId, String taskId, Counters ctrs);
- } catch (Exception e) {
- LOG.warn(org.apache.hadoop.util.StringUtils.stringifyException(e));
- }
- if (sb1.length() > 0) {
- taskInfoMap.get(id).hm.put(Keys.ROWS_INSERTED.name(), sb1.toString());
- queryInfoMap.get(queryId).hm.put(Keys.ROWS_INSERTED.name(), sb1
- .toString());
- }
- if (sb.length() > 0) {
- taskInfoMap.get(id).hm.put(Keys.TASK_COUNTERS.name(), sb.toString());
- }
- }
-
- public void printRowCount(String queryId) {
- QueryInfo ji = queryInfoMap.get(queryId);
- if (ji == null) {
- return;
- }
- for (String tab : ji.rowCountMap.keySet()) {
- console.printInfo(ji.rowCountMap.get(tab) + " Rows loaded to " + tab);
- }
- }
+ public void printRowCount(String queryId);
/**
* Called at the end of Job. A Job is sql query.
*
* @param queryId
*/
- public void endQuery(String queryId) {
- QueryInfo ji = queryInfoMap.get(queryId);
- if (ji == null) {
- return;
- }
- log(RecordTypes.QueryEnd, ji.hm);
- queryInfoMap.remove(queryId);
- }
+ public void endQuery(String queryId);
/**
* Called at the start of a task. Called by Driver.run() A Job can have
@@ -439,108 +167,32 @@ public void endQuery(String queryId) {
* @param task
*/
public void startTask(String queryId, Task extends Serializable> task,
- String taskName) {
- SessionState ss = SessionState.get();
- if (ss == null) {
- return;
- }
- TaskInfo ti = new TaskInfo();
-
- ti.hm.put(Keys.QUERY_ID.name(), ss.getQueryId());
- ti.hm.put(Keys.TASK_ID.name(), task.getId());
- ti.hm.put(Keys.TASK_NAME.name(), taskName);
-
- String id = queryId + ":" + task.getId();
- taskInfoMap.put(id, ti);
-
- log(RecordTypes.TaskStart, ti.hm);
-
- }
+ String taskName);
/**
* Called at the end of a task.
*
* @param task
*/
- public void endTask(String queryId, Task extends Serializable> task) {
- String id = queryId + ":" + task.getId();
- TaskInfo ti = taskInfoMap.get(id);
-
- if (ti == null) {
- return;
- }
- log(RecordTypes.TaskEnd, ti.hm);
- taskInfoMap.remove(id);
- }
+ public void endTask(String queryId, Task extends Serializable> task);
/**
* Called at the end of a task.
*
* @param task
*/
- public void progressTask(String queryId, Task extends Serializable> task) {
- String id = queryId + ":" + task.getId();
- TaskInfo ti = taskInfoMap.get(id);
- if (ti == null) {
- return;
- }
- log(RecordTypes.TaskProgress, ti.hm);
-
- }
-
- /**
- * write out counters.
- */
- static ThreadLocal