commit 75bff8f29c4908b4a4a5cff60d270a3dfad43583 Author: Bharath Krishna Date: Wed May 23 15:54:08 2018 -0700 HIVE-19508 : SparkJobMonitor getReport doesn't print stage progress in order diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java index 7afd8864075aa0d9708274eea8839c662324c732..628226315fd24874b068f638937b5b74fe608b3c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java @@ -30,6 +30,7 @@ import java.text.DecimalFormat; import java.text.NumberFormat; import java.text.SimpleDateFormat; +import java.util.Comparator; import java.util.Date; import java.util.HashSet; import java.util.Map; @@ -77,6 +78,8 @@ private static final int progressBarChars = 30; private final NumberFormat secondsFormat = new DecimalFormat("#0.00"); + private final Comparator compareStageAsInt = + (String s1, String s2) -> (Integer.compare(getStageNum(s1), getStageNum(s2))); protected SparkJobMonitor(HiveConf hiveConf) { monitorTimeoutInterval = hiveConf.getTimeVar( @@ -104,7 +107,8 @@ private void printStatusInPlace(Map progressMap) { reprintLineWithColorAsBold(HEADER, Ansi.Color.CYAN); reprintLine(SEPARATOR); - SortedSet keys = new TreeSet(progressMap.keySet()); + SortedSet keys = new TreeSet(compareStageAsInt); + keys.addAll(progressMap.keySet()); int idx = 0; final int numKey = keys.size(); for (String s : keys) { @@ -193,6 +197,11 @@ protected int getStageMaxTaskCount(Map progressMap) return stageMaxTasks; } + private int getStageNum(String s) { + int div = s.indexOf('_'); + return Integer.parseInt(div > 0 ? s.substring(0, div) : s); + } + private String getReport(Map progressMap) { StringBuilder reportBuffer = new StringBuilder(); SimpleDateFormat dt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); @@ -203,7 +212,9 @@ private String getReport(Map progressMap) { int sumTotal = 0; int sumComplete = 0; - SortedSet keys = new TreeSet(progressMap.keySet()); + SortedSet keys = new TreeSet(compareStageAsInt); + keys.addAll(progressMap.keySet()); + for (String s : keys) { SparkStageProgress progress = progressMap.get(s); final int complete = progress.getSucceededTaskCount();