Details
-
Bug
-
Status: Resolved
-
Blocker
-
Resolution: Fixed
-
3.1.0, 3.1.1, 3.1.2, 3.2.0, 3.1.3, 3.2.1
Description
A performance improvement to how Spark casts Strings to Decimal in this PR title, has introduced a regression
scala> :paste
// Entering paste mode (ctrl-D to finish)
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
val data = Seq(Row("7.836725755512218E38"))
val schema=StructType(Array(StructField("a", StringType, false)))
val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.select(col("a").cast(DecimalType(37,-17))).show
// Exiting paste mode, now interpreting.
+--------------------+
| a|
+--------------------+
|7.836725755512218...|
+--------------------+
scala> spark.version
res2: String = 3.0.1
scala> :paste
// Entering paste mode (ctrl-D to finish)
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
val data = Seq(Row("7.836725755512218E38"))
val schema=StructType(Array(StructField("a", StringType, false)))
val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.select(col("a").cast(DecimalType(37,-17))).show
// Exiting paste mode, now interpreting.
+----+
| a|
+----+
|null|
+----+
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
data: Seq[org.apache.spark.sql.Row] = List([7.836725755512218E38])
schema: org.apache.spark.sql.types.StructType = StructType(StructField(a,StringType,false))
df: org.apache.spark.sql.DataFrame = [a: string]
scala> spark.version
res1: String = 3.1.1
