commit 09776908169f9512d2858cbdcf0ebff4c89d9983 Author: Owen O'Malley Date: Thu Jun 25 11:20:06 2015 -0700 HIVE-10795. Remove PerfLogger from ORC. diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 7346bc4..8864013 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -58,7 +58,6 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; -import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -122,9 +121,6 @@ private static final long DEFAULT_MIN_SPLIT_SIZE = 16 * 1024 * 1024; private static final long DEFAULT_MAX_SPLIT_SIZE = 256 * 1024 * 1024; - private static final PerfLogger perfLogger = PerfLogger.getPerfLogger(); - private static final String CLASS_NAME = ReaderImpl.class.getName(); - /** * When picking the hosts for a split that crosses block boundaries, * any drop any host that has fewer than MIN_INCLUDED_LOCATION of the @@ -490,7 +486,7 @@ private FileInfo verifyCachedFileInfo(FileStatus file) { context.numFilesCounter.incrementAndGet(); FileInfo fileInfo = Context.footerCache.getIfPresent(file.getPath()); if (fileInfo != null) { - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Info cached for path: " + file.getPath()); } if (fileInfo.modificationTime == file.getModificationTime() && @@ -501,7 +497,7 @@ private FileInfo verifyCachedFileInfo(FileStatus file) { } else { // Invalidate Context.footerCache.invalidate(file.getPath()); - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Meta-Info for : " + file.getPath() + " changed. CachedModificationTime: " + fileInfo.modificationTime + ", CurrentModificationTime: " @@ -511,7 +507,7 @@ private FileInfo verifyCachedFileInfo(FileStatus file) { } } } else { - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Info not cached for path: " + file.getPath()); } } @@ -871,7 +867,7 @@ OrcSplit createSplit(long offset, long length, includeStripe[i] = (i >= stripeStats.size()) || isStripeSatisfyPredicate(stripeStats.get(i), sarg, filterColumns); - if (LOG.isDebugEnabled() && !includeStripe[i]) { + if (isDebugEnabled && !includeStripe[i]) { LOG.debug("Eliminating ORC stripe-" + i + " of file '" + file.getPath() + "' as it did not satisfy " + "predicate condition."); @@ -1060,9 +1056,13 @@ private static void cancelFutures(List> futures) { @Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ORC_GET_SPLITS); + if (isDebugEnabled) { + LOG.debug("getSplits started"); + } List result = generateSplitsInfo(job, numSplits); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ORC_GET_SPLITS); + if (isDebugEnabled) { + LOG.debug("getSplits finished"); + } return result.toArray(new InputSplit[result.size()]); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java index b6ad0dc..1833d3d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java @@ -21,10 +21,10 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; @@ -37,8 +37,8 @@ /** An InputFormat for ORC files. Keys are meaningless, * value is the OrcStruct object */ public class OrcNewInputFormat extends InputFormat{ - private static final PerfLogger perfLogger = PerfLogger.getPerfLogger(); - private static final String CLASS_NAME = ReaderImpl.class.getName(); + + private static final Log LOG = LogFactory.getLog(OrcNewInputFormat.class); @Override public RecordReader createRecordReader( @@ -117,7 +117,9 @@ public boolean nextKeyValue() throws IOException, InterruptedException { @Override public List getSplits(JobContext jobContext) throws IOException, InterruptedException { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ORC_GET_SPLITS); + if (LOG.isDebugEnabled()) { + LOG.debug("getSplits started"); + } List splits = OrcInputFormat.generateSplitsInfo(ShimLoader.getHadoopShims() .getConfiguration(jobContext)); @@ -125,7 +127,9 @@ public boolean nextKeyValue() throws IOException, InterruptedException { for(OrcSplit split: splits) { result.add(new OrcNewSplit(split)); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ORC_GET_SPLITS); + if (LOG.isDebugEnabled()) { + LOG.debug("getSplits finished"); + } return result; } diff --git ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index f202991..20ca195 100644 --- ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -63,7 +63,6 @@ public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor"; public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators"; public static final String LOAD_HASHTABLE = "LoadHashtable"; - public static final String ORC_GET_SPLITS = "OrcGetSplits"; public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning"; public static final String SPARK_BUILD_PLAN = "SparkBuildPlan";