Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1357734) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -382,6 +382,12 @@ //Location of Hive run time structured log file HIVEHISTORYFILELOC("hive.querylog.location", "/tmp/" + System.getProperty("user.name")), + // Whether to log the plan's progress every time a job's progress is checked + HIVE_LOG_INCREMENTAL_PLAN_PROGRESS("hive.querylog.enable.plan.progress", true), + + // The interval between logging the plan's progress in milliseconds + HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL("hive.querylog.plan.progress.interval", 60000), + // Default serde and record reader for user scripts HIVESCRIPTSERDE("hive.script.serde", "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"), HIVESCRIPTRECORDREADER("hive.script.recordreader", Index: conf/hive-default.xml.template =================================================================== --- conf/hive-default.xml.template (revision 1357734) +++ conf/hive-default.xml.template (working copy) @@ -708,6 +708,38 @@ + hive.querylog.location + /tmp/${user.name} + + Location of Hive run time structured log file + + + + + hive.querylog.enable.plan.progress + true + + Whether to log the plan's progress every time a job's progress is checked. + These logs are written to the location specified by hive.querylog.location + + + + + hive.querylog.plan.progress.interval + 60000 + + The interval to wait between logging the plan's progress in milliseconds. + If there is a whole number percentage change in the progress of the mappers or the reducers, + the progress is logged regardless of this value. + The actual interval will be the ceiling of (this value divided by the value of + hive.exec.counters.pull.interval) multiplied by the value of hive.exec.counters.pull.interval + I.e. if it is not divide evenly by the value of hive.exec.counters.pull.interval it will be + logged less frequently than specified. + This only has an effect if hive.querylog.enable.plan.progress is set to true. + + + + hive.enforce.bucketing false Whether bucketing is enforced. If true, while inserting into the table, bucketing is enforced. Index: ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java (revision 1357734) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java (working copy) @@ -221,7 +221,8 @@ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); //DecimalFormat longFormatter = new DecimalFormat("###,###"); long reportTime = System.currentTimeMillis(); - long maxReportInterval = 60 * 1000; // One minute + long maxReportInterval = + HiveConf.getLongVar(job, HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL); boolean fatal = false; StringBuilder errMsg = new StringBuilder(); long pullInterval = HiveConf.getLongVar(job, HiveConf.ConfVars.HIVECOUNTERSPULLINTERVAL); @@ -355,8 +356,10 @@ ss.getHiveHistory().setTaskCounters(SessionState.get().getQueryId(), getId(), ctrs); ss.getHiveHistory().setTaskProperty(SessionState.get().getQueryId(), getId(), Keys.TASK_HADOOP_PROGRESS, output); - ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this.task); - this.callBackObj.logPlanProgress(ss); + if (ss.getConf().getBoolVar(HiveConf.ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS)) { + ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this.task); + this.callBackObj.logPlanProgress(ss); + } } console.printInfo(output); lastReport = report;