diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 41d12ce643..1ea5060276 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3365,6 +3365,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_EXEC_INPUT_LISTING_MAX_THREADS("hive.exec.input.listing.max.threads", 0, new SizeValidator(0L, true, 1024L, true), "Maximum number of threads that Hive uses to list file information from file systems (recommended > 1 for blobstore)."), + HIVE_EXECUTION_FINAL_CHECK_MISS_STAGE("hive.execution.final.check.miss.stage", "none", new StringSet("none","check"), + "after execute finished and to check whether all stage executed" ), + /* BLOBSTORE section */ HIVE_BLOBSTORE_SUPPORTED_SCHEMES("hive.blobstore.supported.schemes", "s3,s3a,s3n", diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 0e84b6c9ab..f862fe2f2c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -1908,6 +1908,24 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { } } } + String checkMissLevel = conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_FINAL_CHECK_MISS_STAGE); + if (!"none".equals(checkMissLevel)) { + ArrayList isTraveled = new ArrayList<>(); + if (!checkTaskExecuteFinished(plan.getRootTasks(), isTraveled)) { + SQLState = "08S01"; + errorMessage = "FAILED: Some Execute Stage miss error"; + invokeFailureHooks(perfLogger, hookContext, errorMessage, null); + errorMessage = hookContext.getErrorMessage(); + console.printError(errorMessage); + + // in case we decided to run everything in local mode, restore the + // the jobtracker setting to its initial value + ctx.restoreOriginalTracker(); + + return ErrorMsg.EXECUTION_MISS_STAGE.getErrorCode(); + } + } + perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.RUN_TASKS); // in case we decided to run everything in local mode, restore the @@ -2058,6 +2076,34 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { return (0); } + private boolean checkTaskExecuteFinished(List> rootTasks, List isTraveled) { + boolean ret = true; + if (rootTasks != null) { + for (Task task : rootTasks) { + // avoid repeat check + if (isTraveled.contains(task.getId())) { + continue; + } else { + isTraveled.add(task.getId()); + } + + if (!task.done()) { + return false; + } + + List> childTasks = null; + if (task instanceof ConditionalTask) { + childTasks = ((ConditionalTask)task).getResTasks(); + } else { + childTasks = task.getDependentTasks(); + } + ret &= checkTaskExecuteFinished(childTasks, isTraveled); + } + } + return ret; + } + + private void releasePlan(QueryPlan plan) { // Plan maybe null if Driver.close is called in another thread for the same Driver object lDrvState.stateLock.lock(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 6a433859b8..75312a4f8e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -537,7 +537,11 @@ "are set. Table schema information is required to read ACID tables"), ACID_TABLES_MUST_BE_READ_WITH_ACID_READER(30021, "An ORC ACID reader required to read ACID tables"), ACID_TABLES_MUST_BE_READ_WITH_HIVEINPUTFORMAT(30022, "Must use HiveInputFormat to read ACID tables " + - "(set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat)") + "(set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat)"), + + //======== 50000 from here ========== + EXECUTION_MISS_STAGE(50001, "Some Execute Stage miss error") + ; private int errorCode; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java index 52cb445754..7c0bd4c6ae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java @@ -172,6 +172,10 @@ public void setResolverCtx(Object resolverCtx) { return listTasks; } + public List> getResTasks() { + return resTasks; + } + /** * @param listTasks * the listTasks to set