commit 4fc0e1d4cef7aa6cc61b8bc8a61becf8dcaad864 Author: Chris Drome Date: Tue Jul 26 18:52:14 2016 +0000 HIVE-14344: Intermittent failures caused by leaking delegation tokens diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 25c4514..b390cd3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -69,6 +70,7 @@ import org.apache.tez.dag.api.Edge; import org.apache.tez.dag.api.GroupInputEdge; import org.apache.tez.dag.api.SessionNotRunning; +import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.Vertex; import org.apache.tez.dag.api.VertexGroup; @@ -138,6 +140,10 @@ public int execute(DriverContext driverContext) { // jobConf will hold all the configuration for hadoop, tez, and hive JobConf jobConf = utils.createConfiguration(conf); + // Unset MR/Tez credential paths similar to MAPREDUCE-3727 + TokenCache.cleanUpTokenReferral(jobConf); + jobConf.unset(TezConfiguration.TEZ_CREDENTIALS_PATH); + // Get all user jars from work (e.g. input format stuff). String[] inputOutputJars = work.configureJobConfAndExtractJars(jobConf);