[PIG-5241] Specify the hdfs path directly to spark and avoid the unnecessary download and upload in SparkLauncher.java

• Type: Sub-task
• Status: Open
• Priority: Major
• Resolution: Unresolved
• Fix Version/s: spark-branch
• Component/s: spark
• Assignee: Nándor Kollár
• Reporter: liyunzhang
• Created: 25/May/17 06:27
• Updated: 25/May/17 16:39

Description

//TODO: Specify the hdfs path directly to spark and avoid the unnecessary download and upload in SparkLauncher.java

  private void cacheFiles(String cacheFiles) throws IOException {
        if (cacheFiles != null && !cacheFiles.isEmpty()) {
            File tmpFolder = Files.createTempDirectory("cache").toFile();
            tmpFolder.deleteOnExit();
            for (String file : cacheFiles.split(",")) {
                String fileName = extractFileName(file.trim());
                Path src = new Path(extractFileUrl(file.trim()));
                File tmpFile = new File(tmpFolder, fileName);
                Path tmpFilePath = new Path(tmpFile.getAbsolutePath());
                FileSystem fs = tmpFilePath.getFileSystem(jobConf);
                //TODO: Specify the hdfs path directly to spark and avoid the unnecessary download and upload in SparkLauncher.java
                fs.copyToLocalFile(src, tmpFilePath);
                tmpFile.deleteOnExit();
                LOG.info(String.format("CacheFile:%s", fileName));
                addResourceToSparkJobWorkingDirectory(tmpFile, fileName,
                        ResourceType.FILE);
            }
        }
    }