Uploaded image for project: 'Zeppelin'
  1. Zeppelin
  2. ZEPPELIN-5289

Pyspark interpreter fails according to spark .jar accessing

    XMLWordPrintableJSON

    Details

    • Flags:
      Patch, Important

      Description

      Cannot run pyspark interpreter with spark 3.1.1 using zeppeling 0.9.0

      Output:

      org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
      	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:836)
      	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:744)
      	at org.apache.zeppelin.scheduler.Job.run(Job.java:172)
      	at org.apache.zeppelin.scheduler.AbstractScheduler.runJob(AbstractScheduler.java:132)
      	at org.apache.zeppelin.scheduler.FIFOScheduler.lambda$runJobInScheduler$0(FIFOScheduler.java:42)
      	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
      	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
      	at java.lang.Thread.run(Thread.java:745)
      Caused by: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
      	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:355)
      	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:366)
      	at org.apache.zeppelin.spark.PySparkInterpreter.open(PySparkInterpreter.java:90)
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
      	... 8 more
      Caused by: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:122)
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
      	... 12 more
      Caused by: scala.reflect.internal.FatalError: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
      	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:99)
      	at scala.collection.Iterator.foreach(Iterator.scala:941)
      	at scala.collection.Iterator.foreach$(Iterator.scala:941)
      	at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
      	at scala.collection.IterableLike.foreach(IterableLike.scala:74)
      	at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
      	at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
      	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:87)
      	at scala.tools.nsc.util.ClassPath.list(ClassPath.scala:36)
      	at scala.tools.nsc.util.ClassPath.list$(ClassPath.scala:36)
      	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:30)
      	at scala.tools.nsc.symtab.SymbolLoaders$PackageLoader.doComplete(SymbolLoaders.scala:284)
      	at scala.tools.nsc.symtab.SymbolLoaders$SymbolLoader.complete(SymbolLoaders.scala:230)
      	at scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1542)
      	at scala.reflect.internal.Mirrors$RootsBase.init(Mirrors.scala:257)
      	at scala.tools.nsc.Global.rootMirror$lzycompute(Global.scala:74)
      	at scala.tools.nsc.Global.rootMirror(Global.scala:72)
      	at scala.tools.nsc.Global.rootMirror(Global.scala:44)
      	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass$lzycompute(Definitions.scala:295)
      	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass(Definitions.scala:295)
      	at scala.reflect.internal.Definitions$DefinitionsClass.init(Definitions.scala:1480)
      	at scala.tools.nsc.Global$Run.<init>(Global.scala:1199)
      	at scala.tools.nsc.interpreter.IMain._initialize(IMain.scala:132)
      	at scala.tools.nsc.interpreter.IMain.initializeSynchronous(IMain.scala:154)
      	at org.apache.zeppelin.spark.SparkScala212Interpreter.open(SparkScala212Interpreter.scala:86)
      	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:106)
      	... 13 more
      Caused by: java.io.IOException: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
      	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:141)
      	at scala.reflect.io.FileZipArchive.root$lzycompute(ZipArchive.scala:179)
      	at scala.reflect.io.FileZipArchive.root(ZipArchive.scala:176)
      	at scala.reflect.io.FileZipArchive.allDirs$lzycompute(ZipArchive.scala:212)
      	at scala.reflect.io.FileZipArchive.allDirs(ZipArchive.scala:212)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.findDirEntry(ZipArchiveFileLookup.scala:76)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list(ZipArchiveFileLookup.scala:63)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list$(ZipArchiveFileLookup.scala:62)
      	at scala.tools.nsc.classpath.ZipAndJarClassPathFactory$ZipArchiveClassPath.list(ZipAndJarFileLookupFactory.scala:56)
      	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:91)
      	... 38 more
      Caused by: java.util.zip.ZipException: error in opening zip file
      	at java.util.zip.ZipFile.open(Native Method)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:219)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:149)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:163)
      	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:138)
      	... 47 more
      org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
      	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:836)
      	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:744)
      	at org.apache.zeppelin.scheduler.Job.run(Job.java:172)
      	at org.apache.zeppelin.scheduler.AbstractScheduler.runJob(AbstractScheduler.java:132)
      	at org.apache.zeppelin.scheduler.FIFOScheduler.lambda$runJobInScheduler$0(FIFOScheduler.java:42)
      	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
      	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
      	at java.lang.Thread.run(Thread.java:745)
      Caused by: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
      	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:355)
      	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:366)
      	at org.apache.zeppelin.spark.PySparkInterpreter.open(PySparkInterpreter.java:90)
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
      	... 8 more
      Caused by: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:122)
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
      	... 12 more
      Caused by: scala.reflect.internal.FatalError: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
      	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:99)
      	at scala.collection.Iterator.foreach(Iterator.scala:941)
      	at scala.collection.Iterator.foreach$(Iterator.scala:941)
      	at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
      	at scala.collection.IterableLike.foreach(IterableLike.scala:74)
      	at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
      	at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
      	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:87)
      	at scala.tools.nsc.util.ClassPath.list(ClassPath.scala:36)
      	at scala.tools.nsc.util.ClassPath.list$(ClassPath.scala:36)
      	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:30)
      	at scala.tools.nsc.symtab.SymbolLoaders$PackageLoader.doComplete(SymbolLoaders.scala:284)
      	at scala.tools.nsc.symtab.SymbolLoaders$SymbolLoader.complete(SymbolLoaders.scala:230)
      	at scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1542)
      	at scala.reflect.internal.Mirrors$RootsBase.init(Mirrors.scala:257)
      	at scala.tools.nsc.Global.rootMirror$lzycompute(Global.scala:74)
      	at scala.tools.nsc.Global.rootMirror(Global.scala:72)
      	at scala.tools.nsc.Global.rootMirror(Global.scala:44)
      	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass$lzycompute(Definitions.scala:295)
      	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass(Definitions.scala:295)
      	at scala.reflect.internal.Definitions$DefinitionsClass.init(Definitions.scala:1480)
      	at scala.tools.nsc.Global$Run.<init>(Global.scala:1199)
      	at scala.tools.nsc.interpreter.IMain._initialize(IMain.scala:132)
      	at scala.tools.nsc.interpreter.IMain.initializeSynchronous(IMain.scala:154)
      	at org.apache.zeppelin.spark.SparkScala212Interpreter.open(SparkScala212Interpreter.scala:86)
      	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:106)
      	... 13 more
      Caused by: java.io.IOException: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
      	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:141)
      	at scala.reflect.io.FileZipArchive.root$lzycompute(ZipArchive.scala:179)
      	at scala.reflect.io.FileZipArchive.root(ZipArchive.scala:176)
      	at scala.reflect.io.FileZipArchive.allDirs$lzycompute(ZipArchive.scala:212)
      	at scala.reflect.io.FileZipArchive.allDirs(ZipArchive.scala:212)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.findDirEntry(ZipArchiveFileLookup.scala:76)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list(ZipArchiveFileLookup.scala:63)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list$(ZipArchiveFileLookup.scala:62)
      	at scala.tools.nsc.classpath.ZipAndJarClassPathFactory$ZipArchiveClassPath.list(ZipAndJarFileLookupFactory.scala:56)
      	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:91)
      	... 38 more
      Caused by: java.util.zip.ZipException: error in opening zip file
      	at java.util.zip.ZipFile.open(Native Method)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:219)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:149)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:163)
      	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:138)
      	... 47 more
      

      Interpreter settings:


        Attachments

        1. image-2021-03-15-17-56-51-350.png
          171 kB
          Jeff Zhang
        2. image-2021-03-15-17-57-29-455.png
          59 kB
          Jeff Zhang
        3. interpreter 1.png
          110 kB
          Dmitry Kravchuk
        4. interpreter 2.png
          72 kB
          Dmitry Kravchuk
        5. zeppelin 0.9.0 spark 3.1 interpreter error.png
          69 kB
          Dmitry Kravchuk

          Activity

            People

            • Assignee:
              Unassigned
              Reporter:
              dishka_krauch Dmitry Kravchuk
            • Votes:
              0 Vote for this issue
              Watchers:
              2 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: