Index: ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java (revision 1558961) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestPlan.java (working copy) @@ -25,6 +25,7 @@ import junit.framework.TestCase; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -89,8 +90,8 @@ // store into configuration job.set("fs.default.name", "file:///"); - Utilities.setMapRedWork(job, mrwork, System.getProperty("java.io.tmpdir") + File.separator + - System.getProperty("user.name") + File.separator + "hive"); + Utilities.setMapRedWork(job, mrwork, new Path(System.getProperty("java.io.tmpdir") + File.separator + + System.getProperty("user.name") + File.separator + "hive")); MapredWork mrwork2 = Utilities.getMapRedWork(job); Utilities.clearWork(job); Index: ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java (revision 1558961) +++ ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java (working copy) @@ -81,7 +81,7 @@ pt.put("/tmp/testfolder", partDesc); MapredWork mrwork = new MapredWork(); mrwork.getMapWork().setPathToPartitionInfo(pt); - Utilities.setMapRedWork(job, mrwork,"/tmp/" + System.getProperty("user.name") + "/hive"); + Utilities.setMapRedWork(job, mrwork,new Path("/tmp/" + System.getProperty("user.name"), "hive")); fileSystem = FileSystem.getLocal(conf); testDir = new Path(System.getProperty("test.tmp.dir", System.getProperty( @@ -169,21 +169,18 @@ drv.compile(cmd); //create scratch dir - String emptyScratchDirStr; - Path emptyScratchDir; Context ctx = new Context(newJob); - emptyScratchDirStr = ctx.getMRTmpFileURI(); - emptyScratchDir = new Path(emptyScratchDirStr); + Path emptyScratchDir = ctx.getMRTmpPath(); FileSystem fileSys = emptyScratchDir.getFileSystem(newJob); fileSys.mkdirs(emptyScratchDir); QueryPlan plan = drv.getPlan(); MapRedTask selectTask = (MapRedTask)plan.getRootTasks().get(0); - List inputPaths = Utilities.getInputPaths(newJob, selectTask.getWork().getMapWork(), emptyScratchDir.toString(), ctx); + List inputPaths = Utilities.getInputPaths(newJob, selectTask.getWork().getMapWork(), emptyScratchDir, ctx); Utilities.setInputPaths(newJob, inputPaths); - Utilities.setMapRedWork(newJob, selectTask.getWork(), ctx.getMRTmpFileURI()); + Utilities.setMapRedWork(newJob, selectTask.getWork(), ctx.getMRTmpPath()); CombineHiveInputFormat combineInputFormat = ReflectionUtils.newInstance( CombineHiveInputFormat.class, newJob); Index: ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java (revision 1558961) +++ ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java (working copy) @@ -140,7 +140,7 @@ pt.put("/tmp/testfolder", partDesc); MapredWork mrwork = new MapredWork(); mrwork.getMapWork().setPathToPartitionInfo(pt); - Utilities.setMapRedWork(conf, mrwork,"/tmp/" + System.getProperty("user.name") + "/hive"); + Utilities.setMapRedWork(conf, mrwork,new Path("/tmp/" + System.getProperty("user.name"), "hive")); hiveSplit = new TestHiveInputSplit(); hbsReader = new TestHiveRecordReader(rcfReader, conf); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java (working copy) @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Stack; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; @@ -124,7 +125,7 @@ // generate the temporary file Context baseCtx = parseCtx.getContext(); - String taskTmpDir = baseCtx.getMRTmpFileURI(); + Path taskTmpDir = baseCtx.getMRTmpPath(); // Create the temporary file, its corresponding FileSinkOperaotr, and // its corresponding TableScanOperator. @@ -132,7 +133,7 @@ GenMapRedUtils.createTemporaryFile(parent, child, taskTmpDir, tt_desc, parseCtx); // Add the path to alias mapping - uCtxTask.addTaskTmpDir(taskTmpDir); + uCtxTask.addTaskTmpDir(taskTmpDir.toUri().toString()); uCtxTask.addTTDesc(tt_desc); uCtxTask.addListTopOperators(tableScanOp); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (working copy) @@ -946,12 +946,12 @@ */ protected static TableScanOperator createTemporaryFile( Operator parent, Operator child, - String taskTmpDir, TableDesc tt_desc, ParseContext parseCtx) { + Path taskTmpDir, TableDesc tt_desc, ParseContext parseCtx) { // Create a FileSinkOperator for the file name of taskTmpDir boolean compressIntermediate = parseCtx.getConf().getBoolVar(HiveConf.ConfVars.COMPRESSINTERMEDIATE); - FileSinkDesc desc = new FileSinkDesc(new Path(taskTmpDir), tt_desc, compressIntermediate); + FileSinkDesc desc = new FileSinkDesc(taskTmpDir, tt_desc, compressIntermediate); if (compressIntermediate) { desc.setCompressCodec(parseCtx.getConf().getVar( HiveConf.ConfVars.COMPRESSINTERMEDIATECODEC)); @@ -1008,7 +1008,7 @@ // Generate the temporary file name Context baseCtx = parseCtx.getContext(); - String taskTmpDir = baseCtx.getMRTmpFileURI(); + Path taskTmpDir = baseCtx.getMRTmpPath(); Operator parent = op.getParentOperators().get(0); TableDesc tt_desc = PlanUtils.getIntermediateFileTableDesc(PlanUtils @@ -1023,7 +1023,7 @@ opProcCtx.getMapCurrCtx(); mapCurrCtx.put(tableScanOp, new GenMapRedCtx(childTask, null)); - String streamDesc = taskTmpDir; + String streamDesc = taskTmpDir.toUri().toString(); MapredWork cplan = (MapredWork) childTask.getWork(); if (needsTagging(cplan.getReduceWork())) { @@ -1055,7 +1055,7 @@ } // Add the path to alias mapping - setTaskPlan(taskTmpDir, streamDesc, tableScanOp, cplan.getMapWork(), false, tt_desc); + setTaskPlan(taskTmpDir.toUri().toString(), streamDesc, tableScanOp, cplan.getMapWork(), false, tt_desc); opProcCtx.setCurrTopOp(null); opProcCtx.setCurrAliasId(null); opProcCtx.setCurrTask(childTask); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (working copy) @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Stack; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; @@ -101,10 +102,10 @@ if (localwork != null) { // get the context info and set up the shared tmp URI Context ctx = physicalContext.getContext(); - String tmpFileURI = Utilities.generateTmpURI(ctx.getLocalTmpFileURI(), currTask.getId()); - localwork.setTmpFileURI(tmpFileURI); - String hdfsTmpURI = Utilities.generateTmpURI(ctx.getMRTmpFileURI(), currTask.getId()); - mapredWork.getMapWork().setTmpHDFSFileURI(hdfsTmpURI); + Path tmpPath = Utilities.generateTmpPath(new Path(ctx.getLocalTmpFileURI()), currTask.getId()); + localwork.setTmpPath(tmpPath); + mapredWork.getMapWork().setTmpHDFSPath(Utilities.generateTmpPath( + ctx.getMRTmpPath(), currTask.getId())); // create a task for this local work; right now, this local work is shared // by the original MapredTask and this new generated MapredLocalTask. MapredLocalTask localTask = (MapredLocalTask) TaskFactory.get(localwork, physicalContext @@ -130,7 +131,7 @@ // create new local work and setup the dummy ops MapredLocalWork newLocalWork = new MapredLocalWork(); newLocalWork.setDummyParentOp(dummyOps); - newLocalWork.setTmpFileURI(tmpFileURI); + newLocalWork.setTmpPath(tmpPath); newLocalWork.setInputFileChangeSensitive(localwork.getInputFileChangeSensitive()); newLocalWork.setBucketMapjoinContext(localwork.copyPartSpecMappingOnly()); mapredWork.getMapWork().setMapLocalWork(newLocalWork); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java (working copy) @@ -123,7 +123,7 @@ Task child = children != null && children.size() == 1 ? children.get(0) : null; - String baseTmpDir = parseCtx.getContext().getMRTmpFileURI(); + String baseTmpDir = parseCtx.getContext().getMRTmpPath().toUri().toString(); JoinDesc joinDescriptor = joinOp.getConf(); Map> joinValues = joinDescriptor.getExprs(); Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java (working copy) @@ -238,7 +238,7 @@ for (String path : paths) { PartitionDesc partDesc = work.getPathToPartitionInfo().get(path); PartitionDesc newPartition = changePartitionToMetadataOnly(partDesc); - Path fakePath = new Path(physicalContext.getContext().getMRTmpFileURI() + Path fakePath = new Path(physicalContext.getContext().getMRTmpPath() + newPartition.getTableName() + encode(newPartition.getPartSpec())); work.getPathToPartitionInfo().remove(path); Index: ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java (working copy) @@ -37,7 +37,7 @@ private LinkedHashMap aliasToFetchWork; private boolean inputFileChangeSensitive; private BucketMapJoinContext bucketMapjoinContext; - private String tmpFileURI; + private Path tmpPath; private String stageID; private List> dummyParentOp ; @@ -55,7 +55,7 @@ } public MapredLocalWork(MapredLocalWork clone){ - this.tmpFileURI = clone.tmpFileURI; + this.tmpPath = clone.tmpPath; this.inputFileChangeSensitive=clone.inputFileChangeSensitive; } @@ -151,12 +151,12 @@ return null; } - public void setTmpFileURI(String tmpFileURI) { - this.tmpFileURI = tmpFileURI; + public void setTmpPath(Path tmpPath) { + this.tmpPath = tmpPath; } - public String getTmpFileURI() { - return tmpFileURI; + public Path getTmpPath() { + return tmpPath; } public String getBucketFileName(String bigFileName) { Index: ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (working copy) @@ -83,7 +83,7 @@ new HashMap>(); private MapredLocalWork mapLocalWork; - private String tmpHDFSFileURI; + private Path tmpHDFSPath; private String inputformat; @@ -432,12 +432,12 @@ this.opParseCtxMap = opParseCtxMap; } - public String getTmpHDFSFileURI() { - return tmpHDFSFileURI; + public Path getTmpHDFSPath() { + return tmpHDFSPath; } - public void setTmpHDFSFileURI(String tmpHDFSFileURI) { - this.tmpHDFSFileURI = tmpHDFSFileURI; + public void setTmpHDFSPath(Path tmpHDFSPath) { + this.tmpHDFSPath = tmpHDFSPath; } public void mergingInto(MapWork mapWork) { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -1277,7 +1277,7 @@ } } else { qb.setIsQuery(true); - fname = ctx.getMRTmpFileURI(); + fname = ctx.getMRTmpPath().toString(); ctx.setResDir(new Path(fname)); } } @@ -5472,7 +5472,7 @@ if (isLocal) { // for local directory - we always write to map-red intermediate // store and then copy to local fs - queryTmpdir = new Path(ctx.getMRTmpFileURI()); + queryTmpdir = ctx.getMRTmpPath(); } else { // otherwise write to the file system implied by the directory // no copy is required. we may want to revisit this policy in future Index: ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (working copy) @@ -265,9 +265,9 @@ public void closeOp(boolean abort) throws HiveException { try { if (mapJoinTables != null) { - // get tmp file URI - String tmpURI = this.getExecContext().getLocalWork().getTmpFileURI(); - LOG.info("Temp URI for side table: " + tmpURI); + // get tmp path + Path tmpPath = this.getExecContext().getLocalWork().getTmpPath(); + LOG.info("Temp URI for side table: " + tmpPath); for (byte tag = 0; tag < mapJoinTables.length; tag++) { // get the key and value MapJoinTableContainer tableContainer = mapJoinTables[tag]; @@ -279,10 +279,9 @@ String fileName = getExecContext().getLocalWork().getBucketFileName(bigBucketFileName); // get the tmp URI path; it will be a hdfs path if not local mode String dumpFilePrefix = conf.getDumpFilePrefix(); - String tmpURIPath = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName); - console.printInfo(Utilities.now() + "\tDump the side-table into file: " + tmpURIPath); + Path path = Utilities.generatePath(tmpPath, dumpFilePrefix, tag, fileName); + console.printInfo(Utilities.now() + "\tDump the side-table into file: " + path); // get the hashtable file and path - Path path = new Path(tmpURIPath); FileSystem fs = path.getFileSystem(hconf); ObjectOutputStream out = new ObjectOutputStream(new BufferedOutputStream(fs.create(path), 4096)); try { @@ -291,7 +290,7 @@ out.close(); } tableContainer.clear(); - console.printInfo(Utilities.now() + "\tUpload 1 File to: " + tmpURIPath); + console.printInfo(Utilities.now() + "\tUpload 1 File to: " + path); } } super.closeOp(abort); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy) @@ -557,22 +557,22 @@ } } - public static void setMapRedWork(Configuration conf, MapredWork w, String hiveScratchDir) { + public static void setMapRedWork(Configuration conf, MapredWork w, Path hiveScratchDir) { setMapWork(conf, w.getMapWork(), hiveScratchDir, true); if (w.getReduceWork() != null) { setReduceWork(conf, w.getReduceWork(), hiveScratchDir, true); } } - public static Path setMapWork(Configuration conf, MapWork w, String hiveScratchDir, boolean useCache) { + public static Path setMapWork(Configuration conf, MapWork w, Path hiveScratchDir, boolean useCache) { return setBaseWork(conf, w, hiveScratchDir, MAP_PLAN_NAME, useCache); } - public static Path setReduceWork(Configuration conf, ReduceWork w, String hiveScratchDir, boolean useCache) { + public static Path setReduceWork(Configuration conf, ReduceWork w, Path hiveScratchDir, boolean useCache) { return setBaseWork(conf, w, hiveScratchDir, REDUCE_PLAN_NAME, useCache); } - private static Path setBaseWork(Configuration conf, BaseWork w, String hiveScratchDir, String name, boolean useCache) { + private static Path setBaseWork(Configuration conf, BaseWork w, Path hiveScratchDir, String name, boolean useCache) { try { setPlanPath(conf, hiveScratchDir); @@ -629,7 +629,7 @@ return new Path(planPath, name); } - private static void setPlanPath(Configuration conf, String hiveScratchDir) throws IOException { + private static void setPlanPath(Configuration conf, Path hiveScratchDir) throws IOException { if (getPlanPath(conf) == null) { // this is the unique conf ID, which is kept in JobConf as part of the plan file name String jobID = UUID.randomUUID().toString(); @@ -2466,11 +2466,10 @@ public static String suffix = ".hashtable"; - public static String generatePath(String baseURI, String dumpFilePrefix, + public static Path generatePath(Path basePath, String dumpFilePrefix, Byte tag, String bigBucketFileName) { - String path = new String(baseURI + Path.SEPARATOR + "MapJoin-" + dumpFilePrefix + tag + + return new Path(basePath, "MapJoin-" + dumpFilePrefix + tag + "-" + bigBucketFileName + suffix); - return path; } public static String generateFileName(Byte tag, String bigBucketFileName) { @@ -2478,24 +2477,16 @@ return fileName; } - public static String generateTmpURI(String baseURI, String id) { - String tmpFileURI = new String(baseURI + Path.SEPARATOR + "HashTable-" + id); - return tmpFileURI; + public static Path generateTmpPath(Path basePath, String id) { + return new Path(basePath, "HashTable-" + id); } - public static String generateTarURI(String baseURI, String filename) { - String tmpFileURI = new String(baseURI + Path.SEPARATOR + filename + ".tar.gz"); - return tmpFileURI; + public static Path generateTarPath(Path basePath, String filename) { + return new Path(basePath, filename + ".tar.gz"); } - public static String generateTarURI(Path baseURI, String filename) { - String tmpFileURI = new String(baseURI + Path.SEPARATOR + filename + ".tar.gz"); - return tmpFileURI; - } - public static String generateTarFileName(String name) { - String tmpFileURI = new String(name + ".tar.gz"); - return tmpFileURI; + return name + ".tar.gz"; } public static String generatePath(Path baseURI, String filename) { @@ -2919,7 +2910,7 @@ * @return List of paths to process for the given MapWork * @throws Exception */ - public static List getInputPaths(JobConf job, MapWork work, String hiveScratchDir, Context ctx) + public static List getInputPaths(JobConf job, MapWork work, Path hiveScratchDir, Context ctx) throws Exception { int sequenceNumber = 0; @@ -2974,7 +2965,7 @@ } @SuppressWarnings({"rawtypes", "unchecked"}) - private static Path createEmptyFile(String hiveScratchDir, + private static Path createEmptyFile(Path hiveScratchDir, Class outFileFormat, JobConf job, int sequenceNumber, Properties props, boolean dummyRow) throws IOException, InstantiationException, IllegalAccessException { @@ -2991,7 +2982,6 @@ String newFile = newDir + File.separator + "emptyFile"; Path newFilePath = new Path(newFile); - String onefile = newPath.toString(); FSRecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newFilePath, Text.class, false, props, null); if (dummyRow) { @@ -3007,7 +2997,7 @@ @SuppressWarnings("rawtypes") private static Path createDummyFileForEmptyPartition(Path path, JobConf job, MapWork work, - String hiveScratchDir, String alias, int sequenceNumber) + Path hiveScratchDir, String alias, int sequenceNumber) throws IOException, InstantiationException, IllegalAccessException { String strPath = path.toString(); @@ -3049,7 +3039,7 @@ @SuppressWarnings("rawtypes") private static Path createDummyFileForEmptyTable(JobConf job, MapWork work, - String hiveScratchDir, String alias, int sequenceNumber) + Path hiveScratchDir, String alias, int sequenceNumber) throws IOException, InstantiationException, IllegalAccessException { TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc(); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (working copy) @@ -207,12 +207,11 @@ Path tezDir = getTezDir(mrScratchDir); // set up the operator plan - Path planPath = Utilities.setMapWork(conf, mapWork, - mrScratchDir.toUri().toString(), false); + Utilities.setMapWork(conf, mapWork, mrScratchDir, false); // setup input paths and split info List inputPaths = Utilities.getInputPaths(conf, mapWork, - mrScratchDir.toUri().toString(), ctx); + mrScratchDir, ctx); Utilities.setInputPaths(conf, inputPaths); // create the directories FileSinkOperators need @@ -317,8 +316,7 @@ Path mrScratchDir, Context ctx) throws Exception { // set up operator plan - Path planPath = Utilities.setReduceWork(conf, reduceWork, - mrScratchDir.toUri().toString(), false); + Utilities.setReduceWork(conf, reduceWork, mrScratchDir, false); // create the directories FileSinkOperators need Utilities.createTmpDirs(conf, reduceWork); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (working copy) @@ -203,7 +203,6 @@ Context ctx = driverContext.getCtx(); boolean ctxCreated = false; - String emptyScratchDirStr; Path emptyScratchDir; MapWork mWork = work.getMapWork(); @@ -215,8 +214,7 @@ ctxCreated = true; } - emptyScratchDirStr = ctx.getMRTmpFileURI(); - emptyScratchDir = new Path(emptyScratchDirStr); + emptyScratchDir = ctx.getMRTmpPath(); FileSystem fs = emptyScratchDir.getFileSystem(job); fs.mkdirs(emptyScratchDir); } catch (IOException e) { @@ -331,8 +329,8 @@ MapredLocalWork localwork = mWork.getMapLocalWork(); if (localwork != null) { if (!ShimLoader.getHadoopShims().isLocalMode(job)) { - Path localPath = new Path(localwork.getTmpFileURI()); - Path hdfsPath = new Path(mWork.getTmpHDFSFileURI()); + Path localPath = localwork.getTmpPath(); + Path hdfsPath = mWork.getTmpHDFSPath(); FileSystem hdfs = hdfsPath.getFileSystem(job); FileSystem localFS = localPath.getFileSystem(job); @@ -345,19 +343,16 @@ } //package and compress all the hashtable files to an archive file - String parentDir = localPath.toUri().getPath(); String stageId = this.getId(); - String archiveFileURI = Utilities.generateTarURI(parentDir, stageId); String archiveFileName = Utilities.generateTarFileName(stageId); localwork.setStageID(stageId); - CompressionUtils.tar(parentDir, fileNames,archiveFileName); - Path archivePath = new Path(archiveFileURI); - LOG.info("Archive "+ hashtableFiles.length+" hash table files to " + archiveFileURI); + CompressionUtils.tar(localPath.toUri().getPath(), fileNames,archiveFileName); + Path archivePath = Utilities.generateTarPath(localPath, stageId); + LOG.info("Archive "+ hashtableFiles.length+" hash table files to " + archivePath); //upload archive file to hdfs - String hdfsFile =Utilities.generateTarURI(hdfsPath, stageId); - Path hdfsFilePath = new Path(hdfsFile); + Path hdfsFilePath =Utilities.generateTarPath(hdfsPath, stageId); short replication = (short) job.getInt("mapred.submit.replication", 10); hdfs.setReplication(hdfsFilePath, replication); hdfs.copyFromLocalFile(archivePath, hdfsFilePath); @@ -370,10 +365,10 @@ } } work.configureJobConf(job); - List inputPaths = Utilities.getInputPaths(job, mWork, emptyScratchDirStr, ctx); + List inputPaths = Utilities.getInputPaths(job, mWork, emptyScratchDir, ctx); Utilities.setInputPaths(job, inputPaths); - Utilities.setMapRedWork(job, work, ctx.getMRTmpFileURI()); + Utilities.setMapRedWork(job, work, ctx.getMRTmpPath()); if (mWork.getSamplingType() > 0 && rWork != null && rWork.getNumReduceTasks() > 1) { try { Index: ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java (working copy) @@ -55,13 +55,13 @@ MapJoinTableContainer[] mapJoinTables, MapJoinTableContainerSerDe[] mapJoinTableSerdes) throws HiveException { - String baseDir = null; + Path baseDir = null; Path currentInputPath = context.getCurrentInputPath(); LOG.info("******* Load from HashTable File: input : " + currentInputPath); String fileName = context.getLocalWork().getBucketFileName(currentInputPath.toString()); try { if (ShimLoader.getHadoopShims().isLocalMode(hconf)) { - baseDir = context.getLocalWork().getTmpFileURI(); + baseDir = context.getLocalWork().getTmpPath(); } else { Path[] localArchives; String stageID = context.getLocalWork().getStageID(); @@ -74,8 +74,7 @@ if (!archive.getName().endsWith(suffix)) { continue; } - Path archiveLocalLink = archive.makeQualified(localFs); - baseDir = archiveLocalLink.toUri().getPath(); + baseDir = archive.makeQualified(localFs); } } for (int pos = 0; pos < mapJoinTables.length; pos++) { @@ -85,8 +84,7 @@ if(baseDir == null) { throw new IllegalStateException("baseDir cannot be null"); } - String filePath = Utilities.generatePath(baseDir, desc.getDumpFilePrefix(), (byte)pos, fileName); - Path path = new Path(filePath); + Path path = Utilities.generatePath(baseDir, desc.getDumpFilePrefix(), (byte)pos, fileName); LOG.info("\tLoad back 1 hashtable file from tmp file uri:" + path); ObjectInputStream in = new ObjectInputStream(new BufferedInputStream( new FileInputStream(path.toUri().getPath()), 4096)); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java (working copy) @@ -427,15 +427,14 @@ byte tag = (byte) childOp.getParentOperators().indexOf(parentOp); // generate empty hashtable for this (byte)tag - String tmpURI = this.getWork().getTmpFileURI(); + Path tmpPath = this.getWork().getTmpPath(); String fileName = work.getBucketFileName(bigBucketFileName); HashTableSinkOperator htso = (HashTableSinkOperator)childOp; - String tmpURIPath = Utilities.generatePath(tmpURI, htso.getConf().getDumpFilePrefix(), + Path path = Utilities.generatePath(tmpPath, htso.getConf().getDumpFilePrefix(), tag, fileName); - console.printInfo(Utilities.now() + "\tDump the hashtable into file: " + tmpURIPath); - Path path = new Path(tmpURIPath); + console.printInfo(Utilities.now() + "\tDump the hashtable into file: " + path); FileSystem fs = path.getFileSystem(job); ObjectOutputStream out = new ObjectOutputStream(fs.create(path)); try { @@ -443,7 +442,7 @@ } finally { out.close(); } - console.printInfo(Utilities.now() + "\tUpload 1 File to: " + tmpURIPath + " File size: " + console.printInfo(Utilities.now() + "\tUpload 1 File to: " + path + " File size: " + fs.getFileStatus(path).getLen()); } Index: ql/src/java/org/apache/hadoop/hive/ql/Context.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Context.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/Context.java (working copy) @@ -311,9 +311,9 @@ * * @return next available path for map-red intermediate data */ - public String getMRTmpFileURI() { - return getMRScratchDir() + Path.SEPARATOR + MR_PREFIX + - nextPathId(); + public Path getMRTmpPath() { + return new Path(getMRScratchDir(), MR_PREFIX + + nextPathId()); } Index: ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (working copy) @@ -192,7 +192,7 @@ try { addInputPaths(job, work); - Utilities.setMapWork(job, work, ctx.getMRTmpFileURI(), true); + Utilities.setMapWork(job, work, ctx.getMRTmpPath(), true); // remove the pwd from conf file so that job tracker doesn't show this // logs Index: ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java (working copy) @@ -168,7 +168,7 @@ MapredWork mrWork = new MapredWork(); mrWork.setMapWork(work); - Utilities.setMapRedWork(job, mrWork, ctx.getMRTmpFileURI()); + Utilities.setMapRedWork(job, mrWork, ctx.getMRTmpPath()); // remove the pwd from conf file so that job tracker doesn't show this // logs Index: ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java (working copy) @@ -184,7 +184,7 @@ MapredWork mrWork = new MapredWork(); mrWork.setMapWork(work); - Utilities.setMapRedWork(job, mrWork, ctx.getMRTmpFileURI()); + Utilities.setMapRedWork(job, mrWork, ctx.getMRTmpPath()); // remove the pwd from conf file so that job tracker doesn't show this // logs Index: ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (working copy) @@ -170,7 +170,7 @@ // Build reentrant QL for index query StringBuilder qlCommand = new StringBuilder("INSERT OVERWRITE DIRECTORY "); - String tmpFile = pctx.getContext().getMRTmpFileURI(); + String tmpFile = pctx.getContext().getMRTmpPath().toUri().toString(); queryContext.setIndexIntermediateFile(tmpFile); qlCommand.append( "\"" + tmpFile + "\" "); // QL includes " around file name qlCommand.append("SELECT `_bucketname` , `_offsets` FROM "); Index: ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java (revision 1558961) +++ ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java (working copy) @@ -97,7 +97,7 @@ // Build reentrant QL for index query StringBuilder qlCommand = new StringBuilder("INSERT OVERWRITE DIRECTORY "); - String tmpFile = pctx.getContext().getMRTmpFileURI(); + String tmpFile = pctx.getContext().getMRTmpPath().toUri().toString(); qlCommand.append( "\"" + tmpFile + "\" "); // QL includes " around file name qlCommand.append("SELECT bucketname AS `_bucketname` , COLLECT_SET(offset) AS `_offsets` FROM "); qlCommand.append("(SELECT `_bucketname` AS bucketname , `_offset` AS offset FROM ");