Details
-
Bug
-
Status: Open
-
Major
-
Resolution: Unresolved
-
3.0.1
-
None
-
None
Description
I am getting NotSerializableException when broadcasting Serialized class.
Minimal code with which you can reproduce this issue
case class Student(name: String) class NonSerializable() { def getText() : String = { """ |[ |{ | "name": "test1" |}, |{ | "name": "test2" |} |] |""".stripMargin } } val obj = new NonSerializable() val descriptors_string = obj.getText() import com.github.plokhotnyuk.jsoniter_scala.macros._ import com.github.plokhotnyuk.jsoniter_scala.core._ val parsed_descriptors: Array[Student] = readFromString[Array[Student]](descriptors_string)(JsonCodecMaker.make) val broadcast_descriptors = spark.sparkContext.broadcast(parsed_descriptors) def foo(data: String): Seq[Any] = { import scala.collection.mutable.ArrayBuffer val res = new ArrayBuffer[String]() for (desc <- broadcast_descriptors.value) { res += desc.name } res } val data = spark.sparkContext.parallelize(Array("1", "2", "3")).map(x => foo(x)).collect()
Command used to start spark-shell
./bin/spark-shell --master local --jars /Users/sandeep.katta/.m2/repository/com/github/plokhotnyuk/jsoniter-scala/jsoniter-scala-macros_2.12/2.7.1/jsoniter-scala-macros_2.12-2.7.1.jar,/Users/sandeep.katta/.m2/repository/com/github/plokhotnyuk/jsoniter-scala/jsoniter-scala-core_2.12/2.7.1/jsoniter-scala-core_2.12-2.7.1.jar
Exception Details
org.apache.spark.SparkException: Task not serializable
at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:416)
at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:406)
at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162)
at org.apache.spark.SparkContext.clean(SparkContext.scala:2362)
at org.apache.spark.rdd.RDD.$anonfun$map$1(RDD.scala:396)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:388)
at org.apache.spark.rdd.RDD.map(RDD.scala:395)
... 51 elided
Caused by: java.io.NotSerializableException: NonSerializable
Serialization stack:
- object not serializable (class: NonSerializable, value: NonSerializable@7c95440d)
- field (class: $iw, name: obj, type: class NonSerializable)
- object (class $iw, $iw@3ed476a8)
- field (class: $iw, name: $iw, type: class $iw)
- object (class $iw, $iw@381f7b8e)
- field (class: $iw, name: $iw, type: class $iw)
- object (class $iw, $iw@23635e39)
- field (class: $iw, name: $iw, type: class $iw)
- object (class $iw, $iw@1a8b3791)
- field (class: $iw, name: $iw, type: class $iw)