commit 2d6fbfed374cd1c0e09d42a021b13030dd2b1124 Author: Chris Drome Date: Tue Jul 26 19:02:16 2016 +0000 HIVE-14344: Intermittent failures caused by leaking delegation tokens diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 3c10169..0fcb336 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -62,6 +63,7 @@ import org.apache.tez.dag.api.Edge; import org.apache.tez.dag.api.GroupInputEdge; import org.apache.tez.dag.api.SessionNotRunning; +import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.Vertex; import org.apache.tez.dag.api.VertexGroup; import org.apache.tez.dag.api.client.DAGClient; @@ -122,6 +124,10 @@ public int execute(DriverContext driverContext) { // jobConf will hold all the configuration for hadoop, tez, and hive JobConf jobConf = utils.createConfiguration(conf); + // Unset MR/Tez credential paths similar to MAPREDUCE-3727 + TokenCache.cleanUpTokenReferral(jobConf); + jobConf.unset(TezConfiguration.TEZ_CREDENTIALS_PATH); + // Get all user jars from work (e.g. input format stuff). String[] inputOutputJars = work.configureJobConfAndExtractJars(jobConf);