From ca40ee840bdb06e4d92135de711cf9cfb573d541 Mon Sep 17 00:00:00 2001 From: Rajat Goel Date: Thu, 5 Jan 2012 18:03:55 +0530 Subject: [PATCH] HIVE-2682 [jira] Clean-up logs Summary: Clean-up logs Just wanted to cleanup some logs being printed at wrong loglevel - 1. org.apache.hadoop.hive.ql.exec.CommonJoinOperator prints "table 0 has 1000 rows for join key [...]" as WARNING. Is it really that? 2. org.apache.hadoop.hive.ql.exec.GroupByOperator prints "Hash Table completed flushed" and "Begin Hash Table flush at close: size = 21" as WARNING. It shouldn't be. 4. org.apache.hadoop.hive.ql.stats.jdbc.JDBCStatsPublisher prints "Warning. Invalid statistic." which looks fishy. Test Plan: EMPTY Reviewers: JIRA Differential Revision: https://reviews.facebook.net/D1035 --- .../hadoop/hive/ql/exec/GroupByOperator.java | 8 ++++---- .../apache/hadoop/hive/ql/exec/JoinOperator.java | 7 +++---- .../hive/ql/stats/jdbc/JDBCStatsAggregator.java | 2 +- .../hive/ql/stats/jdbc/JDBCStatsPublisher.java | 3 ++- .../apache/hadoop/hive/serde2/lazy/LazyStruct.java | 2 +- .../hive/serde2/lazybinary/LazyBinaryStruct.java | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java index f8bc71f..073ebb2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java @@ -912,12 +912,12 @@ public class GroupByOperator extends Operator implements } hashAggregations.clear(); hashAggregations = null; - LOG.warn("Hash Table completed flushed"); + LOG.info("Hash Table completed flushed"); return; } int oldSize = hashAggregations.size(); - LOG.warn("Hash Tbl flush: #hash table = " + oldSize); + LOG.info("Hash Tbl flush: #hash table = " + oldSize); Iterator> iter = hashAggregations .entrySet().iterator(); int numDel = 0; @@ -927,7 +927,7 @@ public class GroupByOperator extends Operator implements iter.remove(); numDel++; if (numDel * 10 >= oldSize) { - LOG.warn("Hash Table flushed: new size = " + hashAggregations.size()); + LOG.info("Hash Table flushed: new size = " + hashAggregations.size()); return; } } @@ -995,7 +995,7 @@ public class GroupByOperator extends Operator implements forward(new Object[0], aggregations); } else { if (hashAggregations != null) { - LOG.warn("Begin Hash Table flush at close: size = " + LOG.info("Begin Hash Table flush at close: size = " + hashAggregations.size()); Iterator iter = hashAggregations.entrySet().iterator(); while (iter.hasNext()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java index b9714cc..b914251 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java @@ -109,11 +109,10 @@ public class JoinOperator extends CommonJoinOperator implements } } else { if (sz == nextSz) { - // Output a warning if we reached at least 1000 rows for a join - // operand - // We won't output a warning for the last join operand since the size + // Print a message if we reached at least 1000 rows for a join operand + // We won't print a message for the last join operand since the size // will never goes to joinEmitInterval. - LOG.warn("table " + alias + " has " + sz + " rows for join key " + LOG.info("table " + alias + " has " + sz + " rows for join key " + keyObject); nextSz = getNextSize(nextSz); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java index 48eaf93..df752eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java @@ -115,7 +115,7 @@ public class JDBCStatsAggregator implements StatsAggregator { public String aggregateStats(String fileID, String statType) { if (!JDBCStatsUtils.isValidStatistic(statType)) { - LOG.warn("Warning. Invalid statistic: " + statType + ", supported stats: " + + LOG.warn("Invalid statistic: " + statType + ", supported stats: " + JDBCStatsUtils.getSupportedStatistics()); return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java index b56fe32..aa5c7cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java @@ -125,7 +125,8 @@ public class JDBCStatsPublisher implements StatsPublisher { } if (!JDBCStatsUtils.isValidStatisticSet(stats.keySet())) { - LOG.warn("Warning. Invalid statistic."); + LOG.warn("Invalid statistic:" + stats.keySet().toString() + ", supported " + + " stats: " + JDBCStatsUtils.getSupportedStatistics()); return false; } LOG.info("Stats publishing for key " + fileID); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java index 00ff7af..8a1ea46 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java @@ -164,7 +164,7 @@ public class LazyStruct extends LazyNonPrimitive structByteEnd) { missingFieldWarned = true; - LOG.warn("Missing fields! Expected " + fields.length + " fields but " + LOG.info("Missing fields! Expected " + fields.length + " fields but " + "only got " + fieldId + "! Ignoring similar problems."); } -- 1.7.4.4