diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 605596790c..b387186d34 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -1029,14 +1029,22 @@ public Path getDefaultDestDir(Configuration conf) throws LoginException, IOExcep // reference HDFS based resource directly, to use distribute cache efficiently. addHdfsResource(conf, tmpResources, LocalResourceType.FILE, getHdfsTempFilesFromConf(conf)); // local resources are session based. - addTempResources(conf, tmpResources, hdfsDirPathStr, LocalResourceType.FILE, getLocalTempFilesFromConf(conf), null); + tmpResources.addAll( + addTempResources(conf, hdfsDirPathStr, LocalResourceType.FILE, + getLocalTempFilesFromConf(conf), null).values() + ); } else { // all resources including HDFS are session based. - addTempResources(conf, tmpResources, hdfsDirPathStr, LocalResourceType.FILE, getTempFilesFromConf(conf), null); + tmpResources.addAll( + addTempResources(conf, hdfsDirPathStr, LocalResourceType.FILE, + getTempFilesFromConf(conf), null).values() + ); } - addTempResources(conf, tmpResources, hdfsDirPathStr, LocalResourceType.ARCHIVE, - getTempArchivesFromConf(conf), null); + tmpResources.addAll( + addTempResources(conf, hdfsDirPathStr, LocalResourceType.ARCHIVE, + getTempArchivesFromConf(conf), null).values() + ); return tmpResources; } @@ -1102,26 +1110,22 @@ private void addHdfsResource(Configuration conf, List tmpResource * @param hdfsDirPathStr Destination directory in HDFS. * @param conf Configuration. * @param inputOutputJars The file names to localize. - * @return List<LocalResource> local resources to add to execution + * @return Map<String, LocalResource> (srcPath, local resources) to add to execution * @throws IOException when hdfs operation fails. * @throws LoginException when getDefaultDestDir fails with the same exception */ - public List localizeTempFiles(String hdfsDirPathStr, Configuration conf, + public Map localizeTempFiles(String hdfsDirPathStr, Configuration conf, String[] inputOutputJars, String[] skipJars) throws IOException, LoginException { if (inputOutputJars == null) { return null; } - List tmpResources = new ArrayList(); - addTempResources(conf, tmpResources, hdfsDirPathStr, - LocalResourceType.FILE, inputOutputJars, skipJars); - return tmpResources; + return addTempResources(conf, hdfsDirPathStr, LocalResourceType.FILE, inputOutputJars, skipJars); } - private void addTempResources(Configuration conf, - List tmpResources, String hdfsDirPathStr, - LocalResourceType type, - String[] files, String[] skipFiles) throws IOException { + private Map addTempResources(Configuration conf, String hdfsDirPathStr, + LocalResourceType type, String[] files, String[] skipFiles) throws IOException { HashSet skipFileSet = null; + Map tmpResourcesMap = new HashMap<>(); if (skipFiles != null) { skipFileSet = new HashSet<>(); for (String skipFile : skipFiles) { @@ -1142,8 +1146,9 @@ private void addTempResources(Configuration conf, Path hdfsFilePath = new Path(hdfsDirPathStr, getResourceBaseName(new Path(file))); LocalResource localResource = localizeResource(new Path(file), hdfsFilePath, type, conf); - tmpResources.add(localResource); + tmpResourcesMap.put(file, localResource); } + return tmpResourcesMap; } public FileStatus getHiveJarDirectory(Configuration conf) throws IOException, LoginException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index 767b359219..e251279b1d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -608,7 +608,7 @@ public void ensureLocalResources(Configuration conf, String[] newFilesNotFromCon } // Localize the non-conf resources that are missing from the current list. - List newResources = null; + Map newResources = null; if (newFilesNotFromConf != null && newFilesNotFromConf.length > 0) { boolean hasResources = !resources.additionalFilesNotFromConf.isEmpty(); if (hasResources) { @@ -622,12 +622,8 @@ public void ensureLocalResources(Configuration conf, String[] newFilesNotFromCon if (!hasResources) { String[] skipFilesFromConf = DagUtils.getTempFilesFromConf(conf); newResources = utils.localizeTempFiles(dir, conf, newFilesNotFromConf, skipFilesFromConf); - if (newResources != null) { - resources.localizedResources.addAll(newResources); - } - for (int i=0;i resources = Collections.singletonList(res); + final Map resources = Collections.singletonMap(jarFilePath, res); when(utils.localizeTempFiles(anyString(), any(Configuration.class), eq(inputOutputJars), any(String[].class))).thenReturn(resources);