From 85592dbd942fc450c65fab0cad3b2850984e1db7 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Tue, 11 Oct 2016 14:19:22 -0700 Subject: [PATCH] HIVE-14922 : Add perf logging for post job completion steps --- ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java | 6 ++++-- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 7 +++++++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 4 ++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 6d05f07..ec21cd6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -445,8 +445,10 @@ public int execute(DriverContext driverContext) { SessionState.get().getTxnMgr().getCurrentTxnId(), hasFollowingStatsTask(), work.getLoadTableWork().getWriteType()); - console.printInfo("\t Time taken to load dynamic partitions: " + - (System.currentTimeMillis() - startTime)/1000.0 + " seconds"); + String loadTime = "\t Time taken to load dynamic partitions: " + + (System.currentTimeMillis() - startTime)/1000.0 + " seconds"; + console.printInfo(loadTime); + LOG.info(loadTime); if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) { throw new HiveException("This query creates no partitions." + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index fd64056..a9dbc3e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -1402,16 +1402,23 @@ public static void mvFileToFinalPath(Path specPath, Configuration hconf, FileStatus[] statuses = HiveStatsUtils.getFileStatusRecurse( tmpPath, ((dpCtx == null) ? 1 : dpCtx.getNumDPCols()), fs); if(statuses != null && statuses.length > 0) { + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin("FileSinkOperator", "RemoveTempOrDuplicateFiles"); // remove any tmp file or double-committed output files List emptyBuckets = Utilities.removeTempOrDuplicateFiles(fs, statuses, dpCtx, conf, hconf); + perfLogger.PerfLogEnd("FileSinkOperator", "RemoveTempOrDuplicateFiles"); // create empty buckets if necessary if (emptyBuckets.size() > 0) { + perfLogger.PerfLogBegin("FileSinkOperator", "CreateEmptyBuckets"); createEmptyBuckets(hconf, emptyBuckets, conf, reporter); + perfLogger.PerfLogEnd("FileSinkOperator", "CreateEmptyBuckets"); } // move to the file destination log.info("Moving tmp dir: " + tmpPath + " to: " + specPath); + perfLogger.PerfLogBegin("FileSinkOperator", "RenameOrMoveFiles"); Utilities.renameOrMoveFiles(fs, tmpPath, specPath); + perfLogger.PerfLogEnd("FileSinkOperator", "RenameOrMoveFiles"); } } else { fs.delete(tmpPath, true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 691c3a8..dab4c6a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -128,6 +128,7 @@ import org.apache.hadoop.hive.ql.exec.tez.InPlaceUpdates; import org.apache.hadoop.hive.ql.index.HiveIndexHandler; import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.metastore.SynchronizedMetaStoreClient; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; @@ -1567,6 +1568,8 @@ public Partition loadPartition(Path loadPath, Table tbl, newPartPath = oldPartPath; } List newFiles = null; + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin("MoveTask", "FileMoves"); if (replace || (oldPart == null && !isAcid)) { replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(), isSrcLocal); @@ -1578,6 +1581,7 @@ public Partition loadPartition(Path loadPath, Table tbl, FileSystem fs = tbl.getDataLocation().getFileSystem(conf); Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, newFiles); } + perfLogger.PerfLogEnd("MoveTask", "FileMoves"); Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath); alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString()); validatePartition(newTPart); -- 1.7.12.4 (Apple Git-37)