diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 1df6094dd2..6ece0ba8f0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -2710,7 +2710,9 @@ public static boolean isEmptyPath(Configuration job, Path dirPath) throws IOExce return false; } } catch(FileNotFoundException fnf) { - return true; + LOG.error("FileNotFoundException when listing " + dirPath, fnf); + //to handle cases when fs can genuinely throw FNFE due to permissions + throw fnf; } return true; } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java index 305b467439..81dea60067 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java @@ -34,6 +34,7 @@ import static org.mockito.Mockito.when; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -260,7 +261,7 @@ public void testGetInputPathsWithEmptyPartitions() throws Exception { JobConf jobConf = new JobConf(); Configuration conf = new Configuration(); - Path nonExistentPath1 = new Path(UUID.randomUUID().toString()); + Path part1Path = new Path(UUID.randomUUID().toString()); Path nonExistentPath2 = new Path(UUID.randomUUID().toString()); PartitionDesc mockPartitionDesc = mock(PartitionDesc.class); @@ -275,11 +276,11 @@ public void testGetInputPathsWithEmptyPartitions() throws Exception { mockPartitionDesc).getOutputFileFormatClass(); mapWork1.setPathToAliases(new LinkedHashMap<>( - ImmutableMap.of(nonExistentPath1, Lists.newArrayList(alias1Name)))); + ImmutableMap.of(part1Path, Lists.newArrayList(alias1Name)))); mapWork1.setAliasToWork(new LinkedHashMap<>( ImmutableMap.of(alias1Name, (Operator) mock(Operator.class)))); mapWork1.setPathToPartitionInfo(new LinkedHashMap<>( - ImmutableMap.of(nonExistentPath1, mockPartitionDesc))); + ImmutableMap.of(part1Path, mockPartitionDesc))); mapWork2.setPathToAliases(new LinkedHashMap<>( ImmutableMap.of(nonExistentPath2, Lists.newArrayList(alias2Name)))); @@ -289,24 +290,32 @@ public void testGetInputPathsWithEmptyPartitions() throws Exception { ImmutableMap.of(nonExistentPath2, mockPartitionDesc))); List inputPaths = new ArrayList<>(); + Path scratchDir = new Path(HiveConf.getVar(jobConf, HiveConf.ConfVars.LOCALSCRATCHDIR)); try { - Path scratchDir = new Path(HiveConf.getVar(jobConf, HiveConf.ConfVars.LOCALSCRATCHDIR)); + FileSystem fs = FileSystem.getLocal(jobConf); + fs.mkdirs(part1Path); List inputPaths1 = Utilities.getInputPaths(jobConf, mapWork1, scratchDir, mock(Context.class), false); inputPaths.addAll(inputPaths1); assertEquals(inputPaths1.size(), 1); - assertNotEquals(inputPaths1.get(0), nonExistentPath1); + assertNotEquals(inputPaths1.get(0), part1Path); assertTrue(inputPaths1.get(0).getFileSystem(conf).exists(inputPaths1.get(0))); - assertFalse(nonExistentPath1.getFileSystem(conf).exists(nonExistentPath1)); + } finally { + File file; + for (Path path : inputPaths) { + file = new File(path.toString()); + if (file.exists()) { + file.delete(); + } + } + } - List inputPaths2 = Utilities.getInputPaths(jobConf, mapWork2, scratchDir, - mock(Context.class), false); - inputPaths.addAll(inputPaths2); - assertEquals(inputPaths2.size(), 1); - assertNotEquals(inputPaths2.get(0), nonExistentPath2); - assertTrue(inputPaths2.get(0).getFileSystem(conf).exists(inputPaths2.get(0))); - assertFalse(nonExistentPath2.getFileSystem(conf).exists(nonExistentPath2)); + try { + Utilities.getInputPaths(jobConf, mapWork2, scratchDir, mock(Context.class), false); + fail("FileNotFoundException was expected as part folders were not available"); + } catch(FileNotFoundException fnfe) { + // expected to get fnfe } finally { File file; for (Path path : inputPaths) { @@ -316,6 +325,7 @@ public void testGetInputPathsWithEmptyPartitions() throws Exception { } } } + } /** @@ -326,6 +336,21 @@ public void testGetInputPathsWithEmptyPartitions() throws Exception { */ @Test public void testGetInputPathsWithMultipleThreadsAndEmptyPartitions() throws Exception { + testGetInputPathsWithMultipleThreadsAndEmptyPartitions(true); + try { + testGetInputPathsWithMultipleThreadsAndEmptyPartitions(false); + fail("FileNotFoundException was expected as part folders were not available"); + } catch(ExecutionException e) { + if (!(e.getCause() instanceof FileNotFoundException)) { + fail("FileNotFoundException was expected as part folders were not available; " + + "received " + e.getCause()); + } + } + } + + + public void testGetInputPathsWithMultipleThreadsAndEmptyPartitions(boolean withPartFolders) + throws Exception { int numPartitions = 15; JobConf jobConf = new JobConf(); jobConf.setInt(HiveConf.ConfVars.HIVE_EXEC_INPUT_LISTING_MAX_THREADS.varname, @@ -344,20 +369,22 @@ public void testGetInputPathsWithMultipleThreadsAndEmptyPartitions() throws Exce doReturn(HiveSequenceFileOutputFormat.class).when( mockPartitionDesc).getOutputFileFormatClass(); - - for (int i = 0; i < numPartitions; i++) { - String testPartitionName = "p=" + i; - testPartitionsPaths[i] = new Path(testTablePath, "p=" + i); - mapWork.getPathToAliases().put(testPartitionsPaths[i], Lists.newArrayList(testPartitionName)); - mapWork.getAliasToWork().put(testPartitionName, (Operator) mock(Operator.class)); - mapWork.getPathToPartitionInfo().put(testPartitionsPaths[i], mockPartitionDesc); - - } - FileSystem fs = FileSystem.getLocal(jobConf); try { fs.mkdirs(testTablePath); + + for (int i = 0; i < numPartitions; i++) { + String testPartitionName = "p=" + i; + testPartitionsPaths[i] = new Path(testTablePath, "p=" + i); + if (withPartFolders) { + fs.mkdirs(testPartitionsPaths[i]); + } + mapWork.getPathToAliases().put(testPartitionsPaths[i], Lists.newArrayList(testPartitionName)); + mapWork.getAliasToWork().put(testPartitionName, (Operator) mock(Operator.class)); + mapWork.getPathToPartitionInfo().put(testPartitionsPaths[i], mockPartitionDesc); + } + List inputPaths = Utilities.getInputPaths(jobConf, mapWork, new Path(HiveConf.getVar(jobConf, HiveConf.ConfVars.LOCALSCRATCHDIR)), mock(Context.class), false); assertEquals(inputPaths.size(), numPartitions);