Uploaded image for project: 'Zeppelin'
  1. Zeppelin
  2. ZEPPELIN-5289

Pyspark interpreter fails according to spark .jar accessing

    XMLWordPrintableJSON

Details

    • Patch, Important

    Description

      Cannot run pyspark interpreter with spark 3.1.1 using zeppeling 0.9.0

      Output:

      org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
      	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:836)
      	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:744)
      	at org.apache.zeppelin.scheduler.Job.run(Job.java:172)
      	at org.apache.zeppelin.scheduler.AbstractScheduler.runJob(AbstractScheduler.java:132)
      	at org.apache.zeppelin.scheduler.FIFOScheduler.lambda$runJobInScheduler$0(FIFOScheduler.java:42)
      	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
      	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
      	at java.lang.Thread.run(Thread.java:745)
      Caused by: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
      	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:355)
      	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:366)
      	at org.apache.zeppelin.spark.PySparkInterpreter.open(PySparkInterpreter.java:90)
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
      	... 8 more
      Caused by: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:122)
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
      	... 12 more
      Caused by: scala.reflect.internal.FatalError: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
      	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:99)
      	at scala.collection.Iterator.foreach(Iterator.scala:941)
      	at scala.collection.Iterator.foreach$(Iterator.scala:941)
      	at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
      	at scala.collection.IterableLike.foreach(IterableLike.scala:74)
      	at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
      	at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
      	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:87)
      	at scala.tools.nsc.util.ClassPath.list(ClassPath.scala:36)
      	at scala.tools.nsc.util.ClassPath.list$(ClassPath.scala:36)
      	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:30)
      	at scala.tools.nsc.symtab.SymbolLoaders$PackageLoader.doComplete(SymbolLoaders.scala:284)
      	at scala.tools.nsc.symtab.SymbolLoaders$SymbolLoader.complete(SymbolLoaders.scala:230)
      	at scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1542)
      	at scala.reflect.internal.Mirrors$RootsBase.init(Mirrors.scala:257)
      	at scala.tools.nsc.Global.rootMirror$lzycompute(Global.scala:74)
      	at scala.tools.nsc.Global.rootMirror(Global.scala:72)
      	at scala.tools.nsc.Global.rootMirror(Global.scala:44)
      	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass$lzycompute(Definitions.scala:295)
      	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass(Definitions.scala:295)
      	at scala.reflect.internal.Definitions$DefinitionsClass.init(Definitions.scala:1480)
      	at scala.tools.nsc.Global$Run.<init>(Global.scala:1199)
      	at scala.tools.nsc.interpreter.IMain._initialize(IMain.scala:132)
      	at scala.tools.nsc.interpreter.IMain.initializeSynchronous(IMain.scala:154)
      	at org.apache.zeppelin.spark.SparkScala212Interpreter.open(SparkScala212Interpreter.scala:86)
      	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:106)
      	... 13 more
      Caused by: java.io.IOException: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
      	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:141)
      	at scala.reflect.io.FileZipArchive.root$lzycompute(ZipArchive.scala:179)
      	at scala.reflect.io.FileZipArchive.root(ZipArchive.scala:176)
      	at scala.reflect.io.FileZipArchive.allDirs$lzycompute(ZipArchive.scala:212)
      	at scala.reflect.io.FileZipArchive.allDirs(ZipArchive.scala:212)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.findDirEntry(ZipArchiveFileLookup.scala:76)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list(ZipArchiveFileLookup.scala:63)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list$(ZipArchiveFileLookup.scala:62)
      	at scala.tools.nsc.classpath.ZipAndJarClassPathFactory$ZipArchiveClassPath.list(ZipAndJarFileLookupFactory.scala:56)
      	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:91)
      	... 38 more
      Caused by: java.util.zip.ZipException: error in opening zip file
      	at java.util.zip.ZipFile.open(Native Method)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:219)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:149)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:163)
      	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:138)
      	... 47 more
      org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
      	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:836)
      	at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:744)
      	at org.apache.zeppelin.scheduler.Job.run(Job.java:172)
      	at org.apache.zeppelin.scheduler.AbstractScheduler.runJob(AbstractScheduler.java:132)
      	at org.apache.zeppelin.scheduler.FIFOScheduler.lambda$runJobInScheduler$0(FIFOScheduler.java:42)
      	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
      	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
      	at java.lang.Thread.run(Thread.java:745)
      Caused by: org.apache.zeppelin.interpreter.InterpreterException: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
      	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:355)
      	at org.apache.zeppelin.interpreter.Interpreter.getInterpreterInTheSameSessionByClassName(Interpreter.java:366)
      	at org.apache.zeppelin.spark.PySparkInterpreter.open(PySparkInterpreter.java:90)
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
      	... 8 more
      Caused by: org.apache.zeppelin.interpreter.InterpreterException: Fail to open SparkInterpreter
      	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:122)
      	at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
      	... 12 more
      Caused by: scala.reflect.internal.FatalError: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
      	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:99)
      	at scala.collection.Iterator.foreach(Iterator.scala:941)
      	at scala.collection.Iterator.foreach$(Iterator.scala:941)
      	at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
      	at scala.collection.IterableLike.foreach(IterableLike.scala:74)
      	at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
      	at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
      	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:87)
      	at scala.tools.nsc.util.ClassPath.list(ClassPath.scala:36)
      	at scala.tools.nsc.util.ClassPath.list$(ClassPath.scala:36)
      	at scala.tools.nsc.classpath.AggregateClassPath.list(AggregateClassPath.scala:30)
      	at scala.tools.nsc.symtab.SymbolLoaders$PackageLoader.doComplete(SymbolLoaders.scala:284)
      	at scala.tools.nsc.symtab.SymbolLoaders$SymbolLoader.complete(SymbolLoaders.scala:230)
      	at scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1542)
      	at scala.reflect.internal.Mirrors$RootsBase.init(Mirrors.scala:257)
      	at scala.tools.nsc.Global.rootMirror$lzycompute(Global.scala:74)
      	at scala.tools.nsc.Global.rootMirror(Global.scala:72)
      	at scala.tools.nsc.Global.rootMirror(Global.scala:44)
      	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass$lzycompute(Definitions.scala:295)
      	at scala.reflect.internal.Definitions$DefinitionsClass.ObjectClass(Definitions.scala:295)
      	at scala.reflect.internal.Definitions$DefinitionsClass.init(Definitions.scala:1480)
      	at scala.tools.nsc.Global$Run.<init>(Global.scala:1199)
      	at scala.tools.nsc.interpreter.IMain._initialize(IMain.scala:132)
      	at scala.tools.nsc.interpreter.IMain.initializeSynchronous(IMain.scala:154)
      	at org.apache.zeppelin.spark.SparkScala212Interpreter.open(SparkScala212Interpreter.scala:86)
      	at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:106)
      	... 13 more
      Caused by: java.io.IOException: Error accessing /opt/spark31/spark-3.1.1-bin-hadoop3.2/jars/accessors-smart-1.2.jar
      	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:141)
      	at scala.reflect.io.FileZipArchive.root$lzycompute(ZipArchive.scala:179)
      	at scala.reflect.io.FileZipArchive.root(ZipArchive.scala:176)
      	at scala.reflect.io.FileZipArchive.allDirs$lzycompute(ZipArchive.scala:212)
      	at scala.reflect.io.FileZipArchive.allDirs(ZipArchive.scala:212)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.findDirEntry(ZipArchiveFileLookup.scala:76)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list(ZipArchiveFileLookup.scala:63)
      	at scala.tools.nsc.classpath.ZipArchiveFileLookup.list$(ZipArchiveFileLookup.scala:62)
      	at scala.tools.nsc.classpath.ZipAndJarClassPathFactory$ZipArchiveClassPath.list(ZipAndJarFileLookupFactory.scala:56)
      	at scala.tools.nsc.classpath.AggregateClassPath.$anonfun$list$3(AggregateClassPath.scala:91)
      	... 38 more
      Caused by: java.util.zip.ZipException: error in opening zip file
      	at java.util.zip.ZipFile.open(Native Method)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:219)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:149)
      	at java.util.zip.ZipFile.<init>(ZipFile.java:163)
      	at scala.reflect.io.FileZipArchive.scala$reflect$io$FileZipArchive$$openZipFile(ZipArchive.scala:138)
      	... 47 more
      

      Interpreter settings:


      Attachments

        1. zeppelin 0.9.0 spark 3.1 interpreter error.png
          69 kB
          Dmitry Kravchuk
        2. interpreter 2.png
          72 kB
          Dmitry Kravchuk
        3. interpreter 1.png
          110 kB
          Dmitry Kravchuk
        4. image-2021-03-15-17-57-29-455.png
          59 kB
          Jeff Zhang
        5. image-2021-03-15-17-56-51-350.png
          171 kB
          Jeff Zhang

        Activity

          People

            Unassigned Unassigned
            dishka_krauch Dmitry Kravchuk
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: