Uploaded image for project: 'Spark'
  1. Spark
  2. SPARK-3203

ClassNotFoundException in spark-shell with Cassandra

    XMLWordPrintableJSON

Details

    • Bug
    • Status: Resolved
    • Major
    • Resolution: Duplicate
    • 1.0.2
    • None
    • Spark Core
    • None

    • Ubuntu 12.04, openjdk 64 bit 7u65

    Description

      I am using Spark with as processing engine over cassandra. I have only one master and a worker node.

      I am executing following code in spark-shell :

      sc.stop
      import org.apache.spark.SparkContext
      import org.apache.spark.SparkConf
      import com.datastax.spark.connector._

      val conf = new SparkConf(true).set("spark.cassandra.connection.host", "127.0.0.1")
      val sc = new SparkContext("spark://L-BXP44Z1:7077", "Cassandra Connector Test", conf)
      val rdd = sc.cassandraTable("test", "kv")
      println(rdd.map(_.getInt("value")).sum)

      I am getting following error:

      14/08/25 18:47:17 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address
      14/08/25 18:49:39 INFO CoarseGrainedExecutorBackend: Got assigned task 0
      14/08/25 18:49:39 INFO Executor: Running task ID 0
      14/08/25 18:49:39 ERROR Executor: Exception in task ID 0
      java.lang.ClassNotFoundException: $line29.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$1
      at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
      at java.security.AccessController.doPrivileged(Native Method)
      at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
      at java.lang.Class.forName0(Native Method)
      at java.lang.Class.forName(Class.java:270)
      at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:60)
      at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612)
      at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1771)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370)
      at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:606)
      at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1893)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370)
      at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:63)
      at org.apache.spark.scheduler.ResultTask$.deserializeInfo(ResultTask.scala:61)
      at org.apache.spark.scheduler.ResultTask.readExternal(ResultTask.scala:141)
      at java.io.ObjectInputStream.readExternalData(ObjectInputStream.java:1837)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1796)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370)
      at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:63)
      at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:85)
      at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:165)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
      at java.lang.Thread.run(Thread.java:745)
      14/08/25 18:49:39 INFO CoarseGrainedExecutorBackend: Got assigned task 1
      14/08/25 18:49:39 INFO Executor: Running task ID 1
      14/08/25 18:49:39 ERROR Executor: Exception in task ID 1
      java.lang.ClassNotFoundException: $line29.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$1
      at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
      at java.security.AccessController.doPrivileged(Native Method)
      at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
      at java.lang.Class.forName0(Native Method)
      at java.lang.Class.forName(Class.java:270)
      at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:60)
      at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612)
      at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1771)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370)
      at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:606)
      at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1893)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990)
      at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370)
      at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:63)
      at org.apache.spark.scheduler.ResultTask$.deserializeInfo(ResultTask.scala:61)
      at org.apache.spark.scheduler.ResultTask.readExternal(ResultTask.scala:141)
      at java.io.ObjectInputStream.readExternalData(ObjectInputStream.java:1837)
      at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1796)
      at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350)
      at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370)
      at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:63)
      at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:85)
      at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:165)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
      at java.lang.Thread.run(Thread.java:745)
      14/08/25 18:49:39 INFO CoarseGrainedExecutorBackend: Got assigned task 2

      Attachments

        Issue Links

          duplicates

          Bug - A problem which impairs or prevents the functions of the product. SPARK-6299 ClassNotFoundException in standalone mode when running groupByKey with class defined in REPL.

          • Major - Major loss of function.
          • Resolved

          Activity

            People

              Unassigned Unassigned
              rohitkumarbhagat Rohit Kumar
              Votes:
              3 Vote for this issue
              Watchers:
              6 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved: