Details
-
Sub-task
-
Status: Open
-
Major
-
Resolution: Unresolved
-
None
-
None
Description
//TODO: Specify the hdfs path directly to spark and avoid the unnecessary download and upload in SparkLauncher.java
private void cacheFiles(String cacheFiles) throws IOException { if (cacheFiles != null && !cacheFiles.isEmpty()) { File tmpFolder = Files.createTempDirectory("cache").toFile(); tmpFolder.deleteOnExit(); for (String file : cacheFiles.split(",")) { String fileName = extractFileName(file.trim()); Path src = new Path(extractFileUrl(file.trim())); File tmpFile = new File(tmpFolder, fileName); Path tmpFilePath = new Path(tmpFile.getAbsolutePath()); FileSystem fs = tmpFilePath.getFileSystem(jobConf); //TODO: Specify the hdfs path directly to spark and avoid the unnecessary download and upload in SparkLauncher.java fs.copyToLocalFile(src, tmpFilePath); tmpFile.deleteOnExit(); LOG.info(String.format("CacheFile:%s", fileName)); addResourceToSparkJobWorkingDirectory(tmpFile, fileName, ResourceType.FILE); } } }