diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index ea173c8..4253e23 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1079,6 +1079,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal + " expressed as multiple of Local FS write cost"), HIVE_CBO_COST_MODEL_HDFS_READ("hive.cbo.costmodel.hdfs.read", "1.5", "Default cost of reading a byte from HDFS;" + " expressed as multiple of Local FS read cost"), + HIVE_CBO_SHOW_WARNINGS("hive.cbo.show.warnings", false, + "Toggle display of CBO warnings like missing column stats"), AGGR_JOIN_TRANSPOSE("hive.transpose.aggr.join", false, "push aggregates through join"), SEMIJOIN_CONVERSION("hive.enable.semijoin.conversion", true, "convert group by followed by inner equi join into semijoin"), HIVE_COLUMN_ALIGNMENT("hive.order.columnalignment", true, "Flag to control whether we want to try to align" + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index bbdce63..5281d52 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.URI; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.IdentityHashMap; @@ -125,6 +126,25 @@ private Heartbeater heartbeater; private boolean skipTableMasking; + + // Session visible warning messages emitted during query compile, like missing stats + private List compilerWarningMessages = new ArrayList<>(); + + /** + * ThreadLocal context allows retrieving the context from deep in the call stack + * w/o carrying it through params. + */ + private final static ThreadLocal localContext = new ThreadLocal<>(); + + /** + * Gets the current thread Context. + * The context is installed into the thread by CTOR and removed by clear() + * @return + */ + public static Context getCurrent() { + return localContext.get(); + } + /** * This determines the prefix of the * {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.Phase1Ctx#dest} @@ -192,6 +212,7 @@ public Context(Configuration conf, String executionId) { scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION); stagingDir = HiveConf.getVar(conf, HiveConf.ConfVars.STAGINGDIR); opContext = new CompilationOpContext(); + localContext.set(this); } @@ -614,6 +635,7 @@ public void clear() throws IOException { removeScratchDir(); originalTracker = null; setNeedLockMgr(false); + localContext.remove(); } public DataInput getStream() { @@ -886,4 +908,12 @@ public void resetOpContext(){ opContext = new CompilationOpContext(); sequencer = new AtomicInteger(); } + + public void addCompilerWarningMessage(String message) { + compilerWarningMessages.add(message); + } + + public List getCompilerWarningMessages() { + return compilerWarningMessages; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index efa2bdc..58bed0d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -572,6 +572,13 @@ public void run() { } } } + + boolean showCBOWarnings = HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_SHOW_WARNINGS); + if (showCBOWarnings) { + for(String message : ctx.getCompilerWarningMessages()) { + console.printInfo(message); + } + } return 0; } catch (Exception e) { if (isInterrupted()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java index 009d9e5..4098cf6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -391,6 +392,7 @@ private void updateColStats(Set projIndxLst, boolean allowNullColumnFor noColsMissingStats.getAndAdd(colNamesFailedStats.size()); if (allowNullColumnForMissingStats) { LOG.warn(logMsg); + Context.getCurrent().addCompilerWarningMessage(logMsg); } else { LOG.error(logMsg); throw new RuntimeException(logMsg); diff --git a/ql/src/test/queries/clientpositive/stats_missing_warning.q b/ql/src/test/queries/clientpositive/stats_missing_warning.q new file mode 100644 index 0000000..7216509 --- /dev/null +++ b/ql/src/test/queries/clientpositive/stats_missing_warning.q @@ -0,0 +1,53 @@ +set hive.stats.autogather=false; + +CREATE TABLE IF NOT EXISTS missing_stats_t1 (key STRING, value STRING); +CREATE TABLE IF NOT EXISTS missing_stats_t2 (key STRING, value STRING); +CREATE TABLE IF NOT EXISTS missing_stats_t3 (key STRING, value STRING); + +INSERT INTO missing_stats_t1 (key, value) + SELECT key, value + FROM src; + +INSERT INTO missing_stats_t2 (key, value) + SELECT key, value + FROM src; + +INSERT INTO missing_stats_t3 (key, value) + SELECT key, value + FROM src; + +set hive.session.silent=false; + +-- Default should be FALSE +set hive.cbo.show.warnings=true; + +set hive.cbo.enable=true; + +-- Should print warning +set hive.cbo.show.warnings=true; + +SELECT COUNT(*) +FROM missing_stats_t1 t1 +JOIN missing_stats_t2 t2 ON t1.value = t2.key +JOIN missing_stats_t3 t3 ON t2.key = t3.value; + +-- Should not print warning +set hive.cbo.show.warnings=false; + +SELECT COUNT(*) +FROM missing_stats_t1 t1 +JOIN missing_stats_t2 t2 ON t1.value = t2.key +JOIN missing_stats_t3 t3 ON t2.key = t3.value; + +ANALYZE TABLE missing_stats_t1 COMPUTE STATISTICS FOR COLUMNS; +ANALYZE TABLE missing_stats_t2 COMPUTE STATISTICS FOR COLUMNS; +ANALYZE TABLE missing_stats_t3 COMPUTE STATISTICS FOR COLUMNS; + + +-- Warning should be gone +set hive.cbo.show.warnings=true; + +SELECT COUNT(*) +FROM missing_stats_t1 t1 +JOIN missing_stats_t2 t2 ON t1.value = t2.key +JOIN missing_stats_t3 t3 ON t2.key = t3.value; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/stats_missing_warning.q.out b/ql/src/test/results/clientpositive/stats_missing_warning.q.out new file mode 100644 index 0000000..39bb022 --- /dev/null +++ b/ql/src/test/results/clientpositive/stats_missing_warning.q.out @@ -0,0 +1,327 @@ +PREHOOK: query: CREATE TABLE IF NOT EXISTS missing_stats_t1 (key STRING, value STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@missing_stats_t1 +POSTHOOK: query: CREATE TABLE IF NOT EXISTS missing_stats_t1 (key STRING, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@missing_stats_t1 +PREHOOK: query: CREATE TABLE IF NOT EXISTS missing_stats_t2 (key STRING, value STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@missing_stats_t2 +POSTHOOK: query: CREATE TABLE IF NOT EXISTS missing_stats_t2 (key STRING, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@missing_stats_t2 +PREHOOK: query: CREATE TABLE IF NOT EXISTS missing_stats_t3 (key STRING, value STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@missing_stats_t3 +POSTHOOK: query: CREATE TABLE IF NOT EXISTS missing_stats_t3 (key STRING, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@missing_stats_t3 +PREHOOK: query: INSERT INTO missing_stats_t1 (key, value) + SELECT key, value + FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@missing_stats_t1 +POSTHOOK: query: INSERT INTO missing_stats_t1 (key, value) + SELECT key, value + FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@missing_stats_t1 +POSTHOOK: Lineage: missing_stats_t1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: missing_stats_t1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO missing_stats_t2 (key, value) + SELECT key, value + FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@missing_stats_t2 +POSTHOOK: query: INSERT INTO missing_stats_t2 (key, value) + SELECT key, value + FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@missing_stats_t2 +POSTHOOK: Lineage: missing_stats_t2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: missing_stats_t2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT INTO missing_stats_t3 (key, value) + SELECT key, value + FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@missing_stats_t3 +POSTHOOK: query: INSERT INTO missing_stats_t3 (key, value) + SELECT key, value + FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@missing_stats_t3 +POSTHOOK: Lineage: missing_stats_t3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: missing_stats_t3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +No Stats for default@missing_stats_t1, Columns: value +No Stats for default@missing_stats_t2, Columns: key +No Stats for default@missing_stats_t3, Columns: value +PREHOOK: query: SELECT COUNT(*) +FROM missing_stats_t1 t1 +JOIN missing_stats_t2 t2 ON t1.value = t2.key +JOIN missing_stats_t3 t3 ON t2.key = t3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@missing_stats_t1 +PREHOOK: Input: default@missing_stats_t2 +PREHOOK: Input: default@missing_stats_t3 +#### A masked pattern was here #### +WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. +Query ID = rrusanu_20170127090552_5df14965-089b-4336-bddb-7074eff936aa +Total jobs = 2 +Launching Job 1 out of 2 +Number of reduce tasks not specified. Estimated from input data size: 1 +In order to change the average load for a reducer (in bytes): + set hive.exec.reducers.bytes.per.reducer= +In order to limit the maximum number of reducers: + set hive.exec.reducers.max= +In order to set a constant number of reducers: + set mapreduce.job.reduces= +#### A masked pattern was here #### +Hadoop job information for Stage-1: number of mappers: 3; number of reducers: 1 +2017-01-27 09:06:03,789 Stage-1 map = 0%, reduce = 0% +2017-01-27 09:06:12,056 Stage-1 map = 100%, reduce = 0% +2017-01-27 09:06:19,284 Stage-1 map = 100%, reduce = 100% +#### A masked pattern was here #### +Launching Job 2 out of 2 +Number of reduce tasks determined at compile time: 1 +In order to change the average load for a reducer (in bytes): + set hive.exec.reducers.bytes.per.reducer= +In order to limit the maximum number of reducers: + set hive.exec.reducers.max= +In order to set a constant number of reducers: + set mapreduce.job.reduces= +#### A masked pattern was here #### +Hadoop job information for Stage-2: number of mappers: 1; number of reducers: 1 +2017-01-27 09:06:32,766 Stage-2 map = 0%, reduce = 0% +2017-01-27 09:06:39,965 Stage-2 map = 100%, reduce = 0% +2017-01-27 09:06:47,149 Stage-2 map = 100%, reduce = 100% +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) +FROM missing_stats_t1 t1 +JOIN missing_stats_t2 t2 ON t1.value = t2.key +JOIN missing_stats_t3 t3 ON t2.key = t3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@missing_stats_t1 +POSTHOOK: Input: default@missing_stats_t2 +POSTHOOK: Input: default@missing_stats_t3 +#### A masked pattern was here #### +MapReduce Jobs Launched: +Stage-Stage-1: Map: 3 Reduce: 1 HDFS Read: 41236 HDFS Write: 114 SUCCESS +Stage-Stage-2: Map: 1 Reduce: 1 HDFS Read: 5482 HDFS Write: 101 SUCCESS +Total MapReduce CPU Time Spent: 0 msec +OK +0 +Time taken: 55.917 seconds, Fetched: 1 row(s) +PREHOOK: query: SELECT COUNT(*) +FROM missing_stats_t1 t1 +JOIN missing_stats_t2 t2 ON t1.value = t2.key +JOIN missing_stats_t3 t3 ON t2.key = t3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@missing_stats_t1 +PREHOOK: Input: default@missing_stats_t2 +PREHOOK: Input: default@missing_stats_t3 +#### A masked pattern was here #### +WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. +Query ID = rrusanu_20170127090648_8304e901-1bc0-4cb5-8a5f-414cfc5e0e35 +Total jobs = 2 +Launching Job 1 out of 2 +Number of reduce tasks not specified. Estimated from input data size: 1 +In order to change the average load for a reducer (in bytes): + set hive.exec.reducers.bytes.per.reducer= +In order to limit the maximum number of reducers: + set hive.exec.reducers.max= +In order to set a constant number of reducers: + set mapreduce.job.reduces= +#### A masked pattern was here #### +Hadoop job information for Stage-1: number of mappers: 3; number of reducers: 1 +2017-01-27 09:06:59,897 Stage-1 map = 0%, reduce = 0% +2017-01-27 09:07:08,157 Stage-1 map = 100%, reduce = 0% +2017-01-27 09:07:15,338 Stage-1 map = 100%, reduce = 100% +#### A masked pattern was here #### +Launching Job 2 out of 2 +Number of reduce tasks determined at compile time: 1 +In order to change the average load for a reducer (in bytes): + set hive.exec.reducers.bytes.per.reducer= +In order to limit the maximum number of reducers: + set hive.exec.reducers.max= +In order to set a constant number of reducers: + set mapreduce.job.reduces= +#### A masked pattern was here #### +Hadoop job information for Stage-2: number of mappers: 1; number of reducers: 1 +2017-01-27 09:07:27,837 Stage-2 map = 0%, reduce = 0% +2017-01-27 09:07:34,001 Stage-2 map = 100%, reduce = 0% +2017-01-27 09:07:41,186 Stage-2 map = 100%, reduce = 100% +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) +FROM missing_stats_t1 t1 +JOIN missing_stats_t2 t2 ON t1.value = t2.key +JOIN missing_stats_t3 t3 ON t2.key = t3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@missing_stats_t1 +POSTHOOK: Input: default@missing_stats_t2 +POSTHOOK: Input: default@missing_stats_t3 +#### A masked pattern was here #### +MapReduce Jobs Launched: +Stage-Stage-1: Map: 3 Reduce: 1 HDFS Read: 41237 HDFS Write: 114 SUCCESS +Stage-Stage-2: Map: 1 Reduce: 1 HDFS Read: 5497 HDFS Write: 101 SUCCESS +Total MapReduce CPU Time Spent: 0 msec +OK +0 +Time taken: 54.009 seconds, Fetched: 1 row(s) +PREHOOK: query: ANALYZE TABLE missing_stats_t1 COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: default@missing_stats_t1 +#### A masked pattern was here #### +WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. +Query ID = rrusanu_20170127090742_cac7c06a-d423-4df1-84a6-bacc73aade35 +Total jobs = 1 +Launching Job 1 out of 1 +Number of reduce tasks determined at compile time: 1 +In order to change the average load for a reducer (in bytes): + set hive.exec.reducers.bytes.per.reducer= +In order to limit the maximum number of reducers: + set hive.exec.reducers.max= +In order to set a constant number of reducers: + set mapreduce.job.reduces= +#### A masked pattern was here #### +Hadoop job information for Stage-0: number of mappers: 1; number of reducers: 1 +2017-01-27 09:07:54,085 Stage-0 map = 0%, reduce = 0% +2017-01-27 09:08:01,278 Stage-0 map = 100%, reduce = 0% +2017-01-27 09:08:07,452 Stage-0 map = 100%, reduce = 100% +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE missing_stats_t1 COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@missing_stats_t1 +#### A masked pattern was here #### +MapReduce Jobs Launched: +Stage-Stage-0: Map: 1 Reduce: 1 HDFS Read: 17556 HDFS Write: 977 SUCCESS +Total MapReduce CPU Time Spent: 0 msec +OK +Time taken: 26.307 seconds +PREHOOK: query: ANALYZE TABLE missing_stats_t2 COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: default@missing_stats_t2 +#### A masked pattern was here #### +WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. +Query ID = rrusanu_20170127090808_40955b7e-699e-4e9d-a81a-c53b19df8806 +Total jobs = 1 +Launching Job 1 out of 1 +Number of reduce tasks determined at compile time: 1 +In order to change the average load for a reducer (in bytes): + set hive.exec.reducers.bytes.per.reducer= +In order to limit the maximum number of reducers: + set hive.exec.reducers.max= +In order to set a constant number of reducers: + set mapreduce.job.reduces= +#### A masked pattern was here #### +Hadoop job information for Stage-0: number of mappers: 1; number of reducers: 1 +2017-01-27 09:08:21,071 Stage-0 map = 0%, reduce = 0% +2017-01-27 09:08:27,235 Stage-0 map = 100%, reduce = 0% +2017-01-27 09:08:34,435 Stage-0 map = 100%, reduce = 100% +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE missing_stats_t2 COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@missing_stats_t2 +#### A masked pattern was here #### +MapReduce Jobs Launched: +Stage-Stage-0: Map: 1 Reduce: 1 HDFS Read: 17556 HDFS Write: 977 SUCCESS +Total MapReduce CPU Time Spent: 0 msec +OK +Time taken: 26.953 seconds +PREHOOK: query: ANALYZE TABLE missing_stats_t3 COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: default@missing_stats_t3 +#### A masked pattern was here #### +WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. +Query ID = rrusanu_20170127090835_39b7e701-6938-454f-834a-2f7a64789858 +Total jobs = 1 +Launching Job 1 out of 1 +Number of reduce tasks determined at compile time: 1 +In order to change the average load for a reducer (in bytes): + set hive.exec.reducers.bytes.per.reducer= +In order to limit the maximum number of reducers: + set hive.exec.reducers.max= +In order to set a constant number of reducers: + set mapreduce.job.reduces= +#### A masked pattern was here #### +Hadoop job information for Stage-0: number of mappers: 1; number of reducers: 1 +2017-01-27 09:08:46,755 Stage-0 map = 0%, reduce = 0% +2017-01-27 09:08:53,949 Stage-0 map = 100%, reduce = 0% +2017-01-27 09:09:00,100 Stage-0 map = 100%, reduce = 100% +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE missing_stats_t3 COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@missing_stats_t3 +#### A masked pattern was here #### +MapReduce Jobs Launched: +Stage-Stage-0: Map: 1 Reduce: 1 HDFS Read: 17556 HDFS Write: 977 SUCCESS +Total MapReduce CPU Time Spent: 0 msec +OK +Time taken: 25.663 seconds +PREHOOK: query: SELECT COUNT(*) +FROM missing_stats_t1 t1 +JOIN missing_stats_t2 t2 ON t1.value = t2.key +JOIN missing_stats_t3 t3 ON t2.key = t3.value +PREHOOK: type: QUERY +PREHOOK: Input: default@missing_stats_t1 +PREHOOK: Input: default@missing_stats_t2 +PREHOOK: Input: default@missing_stats_t3 +#### A masked pattern was here #### +WARNING: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases. +Query ID = rrusanu_20170127090901_d57c541a-ad6e-4cf5-bd4f-7b5c67006449 +Total jobs = 2 +Launching Job 1 out of 2 +Number of reduce tasks not specified. Estimated from input data size: 1 +In order to change the average load for a reducer (in bytes): + set hive.exec.reducers.bytes.per.reducer= +In order to limit the maximum number of reducers: + set hive.exec.reducers.max= +In order to set a constant number of reducers: + set mapreduce.job.reduces= +#### A masked pattern was here #### +Hadoop job information for Stage-1: number of mappers: 3; number of reducers: 1 +2017-01-27 09:09:12,886 Stage-1 map = 0%, reduce = 0% +2017-01-27 09:09:21,169 Stage-1 map = 100%, reduce = 0% +2017-01-27 09:09:28,355 Stage-1 map = 100%, reduce = 100% +#### A masked pattern was here #### +Launching Job 2 out of 2 +Number of reduce tasks determined at compile time: 1 +In order to change the average load for a reducer (in bytes): + set hive.exec.reducers.bytes.per.reducer= +In order to limit the maximum number of reducers: + set hive.exec.reducers.max= +In order to set a constant number of reducers: + set mapreduce.job.reduces= +#### A masked pattern was here #### +Hadoop job information for Stage-2: number of mappers: 1; number of reducers: 1 +2017-01-27 09:09:41,815 Stage-2 map = 0%, reduce = 0% +2017-01-27 09:09:47,993 Stage-2 map = 100%, reduce = 0% +2017-01-27 09:09:55,182 Stage-2 map = 100%, reduce = 100% +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) +FROM missing_stats_t1 t1 +JOIN missing_stats_t2 t2 ON t1.value = t2.key +JOIN missing_stats_t3 t3 ON t2.key = t3.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@missing_stats_t1 +POSTHOOK: Input: default@missing_stats_t2 +POSTHOOK: Input: default@missing_stats_t3 +#### A masked pattern was here #### +MapReduce Jobs Launched: +Stage-Stage-1: Map: 3 Reduce: 1 HDFS Read: 41683 HDFS Write: 114 SUCCESS +Stage-Stage-2: Map: 1 Reduce: 1 HDFS Read: 5487 HDFS Write: 101 SUCCESS +Total MapReduce CPU Time Spent: 0 msec +OK +0 +Time taken: 55.066 seconds, Fetched: 1 row(s)