[ZEPPELIN-5289] Pyspark interpreter fails according to spark .jar accessing

• Type: Improvement
• Status: Closed
• Priority: Major
• Resolution: Invalid
• Affects Version/s: 0.9.0
• Fix Version/s: 0.9.1
• Component/s: Interpreters, pySpark, zeppelin-interpreter
• Labels: patch

Environment

• zeppelin 0.9.0
• hadoop 3.2
• spark 3.1.1

• Flags: Patch, Important
• Assignee: Unassigned
• Reporter: Dmitry Kravchuk
• Created: 15/Mar/21 08:43
• Updated: 14/Mar/22 08:28
• Resolved: 02/Apr/21 05:33

Description

Cannot run pyspark interpreter with spark 3.1.1 using zeppeling 0.9.0

Output:

org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:836)
	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:744)
	at org.apache.zeppelin.scheduler.Job.run(Job.java:172)
	at org.apache.zeppelin.scheduler.AbstractScheduler.runJob(AbstractScheduler.java:132)
	at org.apache.zeppelin.scheduler.FIFOScheduler.lambda$runJobInScheduler$0(FIFOScheduler.java:42)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:355)
	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:366)
	at org.apache.zeppelin.spark.PySparkInterpreter.open(PySparkInterpreter.java:90)
	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
	... 8 more
Caused by: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:122)
	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
	... 12 more
Caused by: scala.reflect.internal.FatalError: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:99)
	at scala.collection.Iterator.foreach(Iterator.scala:941)
	at scala.collection.Iterator.foreach$(Iterator.scala:941)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
	at scala.collection.IterableLike.foreach(IterableLike.scala:74)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:87)
	at scala.tools.nsc.util.ClassPath.list(ClassPath.scala:36)
	at scala.tools.nsc.util.ClassPath.list$(ClassPath.scala:36)
	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:30)
	at scala.tools.nsc.symtab.SymbolLoaders$PackageLoader.doComplete(SymbolLoaders.scala:284)
	at scala.tools.nsc.symtab.SymbolLoaders$SymbolLoader.complete(SymbolLoaders.scala:230)
	at scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1542)
	at scala.reflect.internal.Mirrors$RootsBase.init(Mirrors.scala:257)
	at scala.tools.nsc.Global.rootMirror$lzycompute(Global.scala:74)
	at scala.tools.nsc.Global.rootMirror(Global.scala:72)
	at scala.tools.nsc.Global.rootMirror(Global.scala:44)
	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass$lzycompute(Definitions.scala:295)
	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass(Definitions.scala:295)
	at scala.reflect.internal.Definitions$DefinitionsClass.init(Definitions.scala:1480)
	at scala.tools.nsc.Global$Run.<init>(Global.scala:1199)
	at scala.tools.nsc.interpreter.IMain._initialize(IMain.scala:132)
	at scala.tools.nsc.interpreter.IMain.initializeSynchronous(IMain.scala:154)
	at org.apache.zeppelin.spark.SparkScala212Interpreter.open(SparkScala212Interpreter.scala:86)
	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:106)
	... 13 more
Caused by: java.io.IOException: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:141)
	at scala.reflect.io.FileZipArchive.root$lzycompute(ZipArchive.scala:179)
	at scala.reflect.io.FileZipArchive.root(ZipArchive.scala:176)
	at scala.reflect.io.FileZipArchive.allDirs$lzycompute(ZipArchive.scala:212)
	at scala.reflect.io.FileZipArchive.allDirs(ZipArchive.scala:212)
	at scala.tools.nsc.classpath.ZipArchiveFileLookup.findDirEntry(ZipArchiveFileLookup.scala:76)
	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list(ZipArchiveFileLookup.scala:63)
	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list$(ZipArchiveFileLookup.scala:62)
	at scala.tools.nsc.classpath.ZipAndJarClassPathFactory$ZipArchiveClassPath.list(ZipAndJarFileLookupFactory.scala:56)
	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:91)
	... 38 more
Caused by: java.util.zip.ZipException: error in opening zip file
	at java.util.zip.ZipFile.open(Native Method)
	at java.util.zip.ZipFile.<init>(ZipFile.java:219)
	at java.util.zip.ZipFile.<init>(ZipFile.java:149)
	at java.util.zip.ZipFile.<init>(ZipFile.java:163)
	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:138)
	... 47 more
org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:836)
	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:744)
	at org.apache.zeppelin.scheduler.Job.run(Job.java:172)
	at org.apache.zeppelin.scheduler.AbstractScheduler.runJob(AbstractScheduler.java:132)
	at org.apache.zeppelin.scheduler.FIFOScheduler.lambda$runJobInScheduler$0(FIFOScheduler.java:42)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:355)
	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:366)
	at org.apache.zeppelin.spark.PySparkInterpreter.open(PySparkInterpreter.java:90)
	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
	... 8 more
Caused by: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:122)
	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
	... 12 more
Caused by: scala.reflect.internal.FatalError: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:99)
	at scala.collection.Iterator.foreach(Iterator.scala:941)
	at scala.collection.Iterator.foreach$(Iterator.scala:941)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
	at scala.collection.IterableLike.foreach(IterableLike.scala:74)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:87)
	at scala.tools.nsc.util.ClassPath.list(ClassPath.scala:36)
	at scala.tools.nsc.util.ClassPath.list$(ClassPath.scala:36)
	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:30)
	at scala.tools.nsc.symtab.SymbolLoaders$PackageLoader.doComplete(SymbolLoaders.scala:284)
	at scala.tools.nsc.symtab.SymbolLoaders$SymbolLoader.complete(SymbolLoaders.scala:230)
	at scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1542)
	at scala.reflect.internal.Mirrors$RootsBase.init(Mirrors.scala:257)
	at scala.tools.nsc.Global.rootMirror$lzycompute(Global.scala:74)
	at scala.tools.nsc.Global.rootMirror(Global.scala:72)
	at scala.tools.nsc.Global.rootMirror(Global.scala:44)
	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass$lzycompute(Definitions.scala:295)
	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass(Definitions.scala:295)
	at scala.reflect.internal.Definitions$DefinitionsClass.init(Definitions.scala:1480)
	at scala.tools.nsc.Global$Run.<init>(Global.scala:1199)
	at scala.tools.nsc.interpreter.IMain._initialize(IMain.scala:132)
	at scala.tools.nsc.interpreter.IMain.initializeSynchronous(IMain.scala:154)
	at org.apache.zeppelin.spark.SparkScala212Interpreter.open(SparkScala212Interpreter.scala:86)
	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:106)
	... 13 more
Caused by: java.io.IOException: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:141)
	at scala.reflect.io.FileZipArchive.root$lzycompute(ZipArchive.scala:179)
	at scala.reflect.io.FileZipArchive.root(ZipArchive.scala:176)
	at scala.reflect.io.FileZipArchive.allDirs$lzycompute(ZipArchive.scala:212)
	at scala.reflect.io.FileZipArchive.allDirs(ZipArchive.scala:212)
	at scala.tools.nsc.classpath.ZipArchiveFileLookup.findDirEntry(ZipArchiveFileLookup.scala:76)
	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list(ZipArchiveFileLookup.scala:63)
	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list$(ZipArchiveFileLookup.scala:62)
	at scala.tools.nsc.classpath.ZipAndJarClassPathFactory$ZipArchiveClassPath.list(ZipAndJarFileLookupFactory.scala:56)
	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:91)
	... 38 more
Caused by: java.util.zip.ZipException: error in opening zip file
	at java.util.zip.ZipFile.open(Native Method)
	at java.util.zip.ZipFile.<init>(ZipFile.java:219)
	at java.util.zip.ZipFile.<init>(ZipFile.java:149)
	at java.util.zip.ZipFile.<init>(ZipFile.java:163)
	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:138)
	... 47 more

Interpreter settings: