Index: conf/hive-default.xml.template
===================================================================
--- conf/hive-default.xml.template (revision 1360519)
+++ conf/hive-default.xml.template (working copy)
@@ -708,6 +708,38 @@
+ hive.querylog.location
+ /tmp/${user.name}
+
+ Location of Hive run time structured log file
+
+
+
+
+ hive.querylog.enable.plan.progress
+ true
+
+ Whether to log the plan's progress every time a job's progress is checked.
+ These logs are written to the location specified by hive.querylog.location
+
+
+
+
+ hive.querylog.plan.progress.interval
+ 60000
+
+ The interval to wait between logging the plan's progress in milliseconds.
+ If there is a whole number percentage change in the progress of the mappers or the reducers,
+ the progress is logged regardless of this value.
+ The actual interval will be the ceiling of (this value divided by the value of
+ hive.exec.counters.pull.interval) multiplied by the value of hive.exec.counters.pull.interval
+ I.e. if it is not divide evenly by the value of hive.exec.counters.pull.interval it will be
+ logged less frequently than specified.
+ This only has an effect if hive.querylog.enable.plan.progress is set to true.
+
+
+
+
hive.enforce.bucketing
false
Whether bucketing is enforced. If true, while inserting into the table, bucketing is enforced.
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1360519)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -382,6 +382,12 @@
//Location of Hive run time structured log file
HIVEHISTORYFILELOC("hive.querylog.location", "/tmp/" + System.getProperty("user.name")),
+ // Whether to log the plan's progress every time a job's progress is checked
+ HIVE_LOG_INCREMENTAL_PLAN_PROGRESS("hive.querylog.enable.plan.progress", true),
+
+ // The interval between logging the plan's progress in milliseconds
+ HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL("hive.querylog.plan.progress.interval", 60000L),
+
// Default serde and record reader for user scripts
HIVESCRIPTSERDE("hive.script.serde", "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"),
HIVESCRIPTRECORDREADER("hive.script.recordreader",
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java (revision 1360519)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java (working copy)
@@ -221,7 +221,8 @@
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
//DecimalFormat longFormatter = new DecimalFormat("###,###");
long reportTime = System.currentTimeMillis();
- long maxReportInterval = 60 * 1000; // One minute
+ long maxReportInterval =
+ HiveConf.getLongVar(job, HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL);
boolean fatal = false;
StringBuilder errMsg = new StringBuilder();
long pullInterval = HiveConf.getLongVar(job, HiveConf.ConfVars.HIVECOUNTERSPULLINTERVAL);
@@ -355,8 +356,10 @@
ss.getHiveHistory().setTaskCounters(SessionState.get().getQueryId(), getId(), ctrs);
ss.getHiveHistory().setTaskProperty(SessionState.get().getQueryId(), getId(),
Keys.TASK_HADOOP_PROGRESS, output);
- ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this.task);
- this.callBackObj.logPlanProgress(ss);
+ if (ss.getConf().getBoolVar(HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS)) {
+ ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this.task);
+ this.callBackObj.logPlanProgress(ss);
+ }
}
console.printInfo(output);
lastReport = report;