Critical Description
See test case in DatasetSuite
ignore("Dataset should set the resolved encoders internally for maps") { val ds: Dataset[(ClassData, Long)] = Seq(ClassData("one", 1), ClassData("two", 2)).toDS() .map(c => ClassData(c.a, c.b + 1)) .groupBy(p => p).count() checkAnswer( ds, (ClassData("one", 1), 1L), (ClassData("two", 2), 1L)) }
Exception shows:
[info] - Dataset should set the resolved encoders internally for maps *** FAILED *** (2 seconds, 267 milliseconds) [info] org.apache.spark.sql.AnalysisException: Can't extract value from a#12; [info] at org.apache.spark.sql.catalyst.expressions.ExtractValue$.apply(complexTypeExtractors.scala:73) [info] at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$apply$10$$anonfun$applyOrElse$4.applyOrElse(Analyzer.scala:471) [info] at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$apply$10$$anonfun$applyOrElse$4.applyOrElse(Analyzer.scala:463) [info] at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:294) [info] at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:294) [info] at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:51) [info] at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:293) [info] at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.apply(TreeNode.scala:291) [info] at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.apply(TreeNode.scala:291) [info] at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:250) [info] at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) [info] at scala.collection.Iterator$class.foreach(Iterator.scala:727) [info] at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) [info] at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) [info] at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) [info] at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) [info] at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273) [info] at scala.collection.AbstractIterator.to(Iterator.scala:1157) [info] at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) [info] at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157) [info] at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252) [info] at scala.collection.AbstractIterator.toArray(Iterator.scala:1157) [info] at org.apache.spark.sql.catalyst.trees.TreeNode.transformChildren(TreeNode.scala:280) [info] at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:291) [info] at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.apply(TreeNode.scala:291) [info] at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.apply(TreeNode.scala:291) [info] at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4$$anonfun$apply$7.apply(TreeNode.scala:269) [info] at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) [info] at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) [info] at scala.collection.immutable.List.foreach(List.scala:318) [info] at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) [info] at scala.collection.AbstractTraversable.map(Traversable.scala:105) [info] at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:267)
