Index: modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java (revision 2474e2b331da34e16acc7a69c4703335e67b142e) +++ modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoaderTest.java (revision ) @@ -94,7 +94,7 @@ CircularWithoutHadoop.class, }; - for (Class c: positiveClasses) + for (Class c : positiveClasses) assertTrue(c.getName(), ldr.hasExternalDependencies(c.getName())); // Negative cases: @@ -104,7 +104,7 @@ Without.class, }; - for (Class c: negativeClasses) + for (Class c : negativeClasses) assertFalse(c.getName(), ldr.hasExternalDependencies(c.getName())); } } \ No newline at end of file Index: modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java (revision 2474e2b331da34e16acc7a69c4703335e67b142e) +++ modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java (revision ) @@ -61,13 +61,11 @@ * unavailable for parent. */ public class HadoopClassLoader extends URLClassLoader implements ClassCache { - static { - // We are very parallel capable. - registerAsParallelCapable(); - } + /** Name of the Hadoop shutdown hook manager class. */ + public static final String SHUTDOWN_HOOK_MGR_CLS_NAME = "org.apache.hadoop.util.ShutdownHookManager"; /** Name of the Hadoop daemon class. */ - public static final String HADOOP_DAEMON_CLASS_NAME = "org.apache.hadoop.util.Daemon"; + public static final String HADOOP_DAEMON_CLS_NAME = "org.apache.hadoop.util.Daemon"; /** Name of libhadoop library. */ private static final String LIBHADOOP = "hadoop."; @@ -97,6 +95,11 @@ /** Native library names. */ private final String[] libNames; + static { + // We are very parallel capable. + registerAsParallelCapable(); + } + /** * Gets name for Job class loader. The name is specific for local node id. * @param locNodeId The local node id. @@ -192,58 +195,20 @@ } } - /** - * Need to parse only Ignite Hadoop and IGFS classes. - * - * @param cls Class name. - * @return {@code true} if we need to check this class. - */ - private static boolean isHadoopIgfs(String cls) { - String ignitePkgPrefix = "org.apache.ignite"; - - int len = ignitePkgPrefix.length(); - - return cls.startsWith(ignitePkgPrefix) && ( - cls.indexOf("igfs.", len) != -1 || - cls.indexOf(".fs.", len) != -1 || - cls.indexOf("hadoop.", len) != -1); - } - - /** - * @param cls Class name. - * @return {@code true} If this is Hadoop class. - */ - private static boolean isHadoop(String cls) { - return cls.startsWith("org.apache.hadoop."); - } - /** {@inheritDoc} */ @Override protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { try { - if (isHadoop(name)) { // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath. - if (name.endsWith(".util.ShutdownHookManager")) // Dirty hack to get rid of Hadoop shutdown hooks. + if (name.equals(SHUTDOWN_HOOK_MGR_CLS_NAME)) + // Dirty hack to get rid of Hadoop shutdown hooks. - return loadFromBytes(name, HadoopShutdownHookManager.class.getName()); + return loadFromBytes(name, HadoopShutdownHookManager.class.getName()); - else if (name.equals(HADOOP_DAEMON_CLASS_NAME)) + else if (name.equals(HADOOP_DAEMON_CLS_NAME)) - // We replace this in order to be able to forcibly stop some daemon threads - // that otherwise never stop (e.g. PeerCache runnables): - return loadFromBytes(name, HadoopDaemon.class.getName()); + // We replace this in order to be able to forcibly stop some daemon threads + // that otherwise never stop (e.g. PeerCache runnables): + return loadFromBytes(name, HadoopDaemon.class.getName()); + if (hasExternalDependencies(name)) return loadClassExplicitly(name, resolve); - } - if (isHadoopIgfs(name)) { // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop. - Boolean hasDeps = cache.get(name); - - if (hasDeps == null) { - hasDeps = hasExternalDependencies(name); - - cache.put(name, hasDeps); - } - - if (hasDeps) - return loadClassExplicitly(name, resolve); - } - return super.loadClass(name, resolve); } catch (NoClassDefFoundError | ClassNotFoundException e) { @@ -362,43 +327,68 @@ * @return {@code True} if class has external dependencies. */ boolean hasExternalDependencies(String clsName) { + Boolean res = cache.get(clsName); + + if (res == null) { - CollectingContext ctx = new CollectingContext(); + CollectingContext ctx = new CollectingContext(); - ctx.annVisitor = new CollectingAnnotationVisitor(ctx); - ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor); - ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor); - ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor); + ctx.annVisitor = new CollectingAnnotationVisitor(ctx); + ctx.mthdVisitor = new CollectingMethodVisitor(ctx, ctx.annVisitor); + ctx.fldVisitor = new CollectingFieldVisitor(ctx, ctx.annVisitor); + ctx.clsVisitor = new CollectingClassVisitor(ctx, ctx.annVisitor, ctx.mthdVisitor, ctx.fldVisitor); - return hasExternalDependencies(clsName, ctx); - } + return hasExternalDependencies(clsName, ctx); + } + else + return res; + } /** * Check whether class has external dependencies on Hadoop. * * @param clsName Class name. * @param ctx Context. + * @return {@code True} if class has external dependencies. + */ + private boolean hasExternalDependencies(String clsName, CollectingContext ctx) { + Boolean res = cache.get(clsName); + + if (res == null) { + res = hasExternalDependencies0(clsName, ctx); + + cache.put(clsName, res); + } + + return res; + } + + /** + * Check whether class has external dependencies on Hadoop. + * + * @param clsName Class name. + * @param ctx Context. * @return {@code true} If the class has external dependencies. */ - boolean hasExternalDependencies(String clsName, CollectingContext ctx) { - if (isHadoop(clsName)) // Hadoop must not be in classpath but Idea sucks, so filtering explicitly as external. - return true; + private boolean hasExternalDependencies0(String clsName, CollectingContext ctx) { + Boolean res = hasDependencyPredefined(clsName); + if (res != null) + return res; + // Try to get from parent to check if the type accessible. InputStream in = loadClassBytes(getParent(), clsName); if (in == null) // The class is external itself, it must be loaded from this class loader. return true; - if (!isHadoopIgfs(clsName)) // Other classes should not have external dependencies. - return false; - final ClassReader rdr; try { rdr = new ClassReader(in); } - catch (IOException e) { - throw new RuntimeException("Failed to read class: " + clsName, e); + catch (Exception e) { + // Most probably we tried to load a class from rt.jar, so ignore. + return false; } ctx.visited.add(clsName); @@ -416,17 +406,73 @@ String parentCls = clsName.substring(0, idx); - if (ctx.visited.contains(parentCls)) - return false; + return hasExternalDependencies(parentCls, ctx); + } - Boolean res = cache.get(parentCls); + /** + * Check whether class name starts with particular prefix. + * + * @param cls Class name. + * @param skip Prefix to skip. + * @param prefix Prefix. + * @return {@code True} if starts with. + */ + private static boolean startsWith(String cls, String skip, String prefix) { + assert cls.startsWith(skip); - if (res == null) - res = hasExternalDependencies(parentCls, ctx); + return cls.startsWith(prefix, skip.length()); + } - return res; + private static final String PKG_ORG = "org."; + + private static final String PKG_ORG_APACHE = "org.apache."; + + /** + * Whether we know in advance whether class has dependency or not. + * + * @param cls Class. + * @return Result. + */ + @Nullable private static Boolean hasDependencyPredefined(String cls) { + // Large "org" group. + if (cls.startsWith(PKG_ORG)) { + // Large "apache" group. + if (startsWith(cls, PKG_ORG, "apache.")) { + // Hadoop classes always have dependencies. + if (startsWith(cls, PKG_ORG_APACHE, "hadoop.")) + return true; + // Filter out Ignite classes which definitely do not have dependencies. + else if (startsWith(cls, PKG_ORG_APACHE, "ignite.")) { + if (!cls.contains(".hadoop.") && !cls.contains(".igfs.") && !cls.contains(".fs.")) + return false; - } + } + // Other well-known "org.apache" packages. + else if (startsWith(cls, PKG_ORG_APACHE, "commons.") || + startsWith(cls, PKG_ORG_APACHE, "log4j.") || + startsWith(cls, PKG_ORG_APACHE, "xerces.")) + return false; + } + // Other well-known "org" packages. + else if (startsWith(cls, PKG_ORG, "jsr166.") || + startsWith(cls, PKG_ORG, "w3c.") || + startsWith(cls, PKG_ORG, "slf4j.") || + startsWith(cls, PKG_ORG, "xml.sax.")) + return false; + } + // Filter out Java system packages. + else if (cls.startsWith("java.") || + cls.startsWith("javax.") || + cls.startsWith("sun.") || + cls.startsWith("com.sun.")) + return false; + // Other well-known packages. + else if (cls.startsWith("com.google.common")) + return false; + // Will parse class. + return null; + } + /** * @param name Class name. * @return {@code true} If this is a valid class name. @@ -571,9 +617,6 @@ assert depCls.indexOf('/') == -1 : depCls; // class name should be fully converted to dot notation. assert depCls.charAt(0) != 'L' : depCls; assert validateClassName(depCls) : depCls; - - if (depCls.startsWith("java.") || depCls.startsWith("javax.")) // Filter out platform classes. - return; if (visited.contains(depCls)) return; \ No newline at end of file Index: modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java (revision 2474e2b331da34e16acc7a69c4703335e67b142e) +++ modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java (revision ) @@ -382,7 +382,7 @@ * @throws Exception On error. */ private void stopHadoopFsDaemons(ClassLoader ldr) throws Exception { - Class daemonCls = ldr.loadClass(HadoopClassLoader.HADOOP_DAEMON_CLASS_NAME); + Class daemonCls = ldr.loadClass(HadoopClassLoader.HADOOP_DAEMON_CLS_NAME); Method m = daemonCls.getMethod("dequeueAndStopAll"); \ No newline at end of file