diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 2b05293..c7f921c 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -62,6 +62,13 @@ * "src\winutils\common.h" * */ public static final int SYMLINK_NO_PRIVILEGE = 2; + + /** + * Variable to hold unexpanded wildcard entries from the classpath + * (created during manifest jar creation during container launch) + */ + public static final String UNEXPANDED_WILDCARD_CLASSPATH = + "UNEXPANDED_WILDCARD_CLASSPATH"; /** * convert an array of FileStatus to an array of Path @@ -1235,6 +1242,7 @@ public static String createJarWithClassPath(String inputClassPath, Path pwd, LOG.debug("mkdirs false for " + workingDir + ", execution will continue"); } + StringBuilder unexpandedWildcardClasspath = new StringBuilder(); // Append all entries List classPathEntryList = new ArrayList( classPathEntries.length); @@ -1243,16 +1251,22 @@ public static String createJarWithClassPath(String inputClassPath, Path pwd, continue; } if (classPathEntry.endsWith("*")) { + boolean foundWildCardJar = false; // Append all jars that match the wildcard Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}"); FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util() .globStatus(globPath); if (wildcardJars != null) { for (FileStatus wildcardJar: wildcardJars) { + foundWildCardJar = true; classPathEntryList.add(wildcardJar.getPath().toUri().toURL() .toExternalForm()); } } + if (!foundWildCardJar) { + unexpandedWildcardClasspath.append(File.pathSeparator); + unexpandedWildcardClasspath.append(classPathEntry); + } } else { // Append just this entry File fileCpEntry = null; @@ -1300,7 +1314,11 @@ public static String createJarWithClassPath(String inputClassPath, Path pwd, } finally { IOUtils.cleanup(LOG, jos, bos, fos); } - + + if (unexpandedWildcardClasspath.length() > 0) { + callerEnv.put(UNEXPANDED_WILDCARD_CLASSPATH, + unexpandedWildcardClasspath.toString()); + } return classPathJar.getCanonicalPath(); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index fc6c64f..cec938f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -150,8 +150,14 @@ public void startLocalizer(Path nmPrivateContainerTokens, Map env = new HashMap(System.getenv()); String classPathJar = FileUtil.createJarWithClassPath(classPath, appStorageDir, env); localizeClasspathJar(new Path(classPathJar), user); + String replacementClassPath = classPathJar; + String unexpandedWildcardClasspath = env.get( + FileUtil.UNEXPANDED_WILDCARD_CLASSPATH); + if (unexpandedWildcardClasspath != null) { + replacementClassPath = replacementClassPath + unexpandedWildcardClasspath; + } command.add("-classpath"); - command.add(classPathJar); + command.add(replacementClassPath); String javaLibPath = System.getProperty("java.library.path"); if (javaLibPath != null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 87a36c4..3f90f49 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -768,7 +768,13 @@ public void sanitizeEnv(Map environment, Path pwd, newClassPath.toString(), pwd, mergedEnv); // In a secure cluster the classpath jar must be localized to grant access this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser()); - environment.put(Environment.CLASSPATH.name(), classPathJar); + String replacementClassPath = classPathJar; + String unexpandedWildcardClasspath = mergedEnv.get( + FileUtil.UNEXPANDED_WILDCARD_CLASSPATH); + if (unexpandedWildcardClasspath != null) { + replacementClassPath = replacementClassPath + unexpandedWildcardClasspath; + } + environment.put(Environment.CLASSPATH.name(), replacementClassPath); } } // put AuxiliaryService data to environment