From 9ac08a0382da82187ef4ec337b4f10b424f41c9a Mon Sep 17 00:00:00 2001 From: xzn Date: Thu, 29 Jul 2021 13:11:42 +0800 Subject: [PATCH] add token for fsScratchDirs to avoid token not found --- .../apache/hadoop/hive/ql/exec/Utilities.java | 16 ++++++++++++++++ .../hadoop/hive/ql/exec/mr/ExecDriver.java | 2 ++ 2 files changed, 18 insertions(+) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 97901234476..bb400686b86 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -197,6 +197,7 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.alias.CredentialProviderFactory; @@ -4990,4 +4991,19 @@ public static boolean arePathsEqualOrWithin(Path p1, Path p2) { return ((p1.toString().toLowerCase().indexOf(p2.toString().toLowerCase()) > -1) || (p2.toString().toLowerCase().indexOf(p1.toString().toLowerCase()) > -1)) ? true : false; } + + /** + * Convenience method to obtain delegation tokens + * corresponding to the paths passed for mapReduce job. + * @param job jonconf + * @param ps array of paths + */ + public static void setToken(JobConf job, Path[] ps) { + try { + TokenCache.obtainTokensForNamenodes(job.getCredentials(), + ps, job); + } catch (IOException ex) { + LOG.error("Error in setToken ", ex); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index ce12cea66ed..5ccab98a482 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -354,6 +354,8 @@ public int execute() { Utilities.setMapRedWork(job, work, ctx.getMRTmpPath()); + Utilities.setToken(job, ctx.getFsScratchDirs().values().toArray(new Path[0])); + if (mWork.getSamplingType() > 0 && rWork != null && job.getNumReduceTasks() > 1) { try { handleSampling(ctx, mWork, job);