[SPARK-829] scheduler shouldn't hang if a task contains unserializable objects in its closure

• Type: Bug
• Status: Resolved
• Priority: Critical
• Resolution: Fixed
• Fix Version/s: 0.8.0
• Assignee: Reynold Xin
• Reporter: Reynold Xin
• Created: 23/Jul/13 14:00
• Updated: 05/Aug/13 16:31
• Resolved: 05/Aug/13 16:31

Description

Try

val a = new Object
sc.parallelize(1 to 10, 2).map(x => a).count

The following exception is uncaught:

java.io.NotSerializableException: java.lang.Object
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1165)
	at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1531)
	at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1413)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1531)
	at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1413)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1531)
	at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1413)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1531)
	at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1413)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1531)
	at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1413)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1531)
	at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1413)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1531)
	at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1413)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1531)
	at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1413)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1531)
	at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1413)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:331)
	at spark.JavaSerializationStream.writeObject(JavaSerializer.scala:28)
	at spark.scheduler.ResultTask$.serializeInfo(ResultTask.scala:43)
	at spark.scheduler.ResultTask.writeExternal(ResultTask.scala:115)
	at java.io.ObjectOutputStream.writeExternalData(ObjectOutputStream.java:1442)
	at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1411)
	at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1159)
	at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:331)
	at spark.JavaSerializationStream.writeObject(JavaSerializer.scala:28)
	at spark.JavaSerializerInstance.serialize(JavaSerializer.scala:48)
	at spark.scheduler.Task$.serializeWithDependencies(Task.scala:78)
	at spark.scheduler.local.LocalTaskSetManager.slaveOffer(LocalTaskSetManager.scala:115)
	at spark.scheduler.local.LocalScheduler$$anonfun$resourceOffer$2.apply(LocalScheduler.scala:134)
	at spark.scheduler.local.LocalScheduler$$anonfun$resourceOffer$2.apply(LocalScheduler.scala:131)
	at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:60)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
	at spark.scheduler.local.LocalScheduler.resourceOffer(LocalScheduler.scala:131)
	at spark.scheduler.local.LocalActor$$anonfun$receive$1.apply(LocalScheduler.scala:46)
	at spark.scheduler.local.LocalActor$$anonfun$receive$1.apply(LocalScheduler.scala:44)
	at akka.actor.Actor$class.apply(Actor.scala:318)
	at spark.scheduler.local.LocalActor.apply(LocalScheduler.scala:43)
	at akka.actor.ActorCell.invoke(ActorCell.scala:626)
	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:197)
	at akka.dispatch.Mailbox.run(Mailbox.scala:179)
	at akka.dispatch.ForkJoinExecutorConfigurator$MailboxExecutionTask.exec(AbstractDispatcher.scala:516)
	at akka.jsr166y.ForkJoinTask.doExec(ForkJoinTask.java:259)
	at akka.jsr166y.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:975)
	at akka.jsr166y.ForkJoinPool.runWorker(ForkJoinPool.java:1479)
	at akka.jsr166y.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:104)