Major Description
import numpy as np from pyspark.sql.functions import lit dtype_to_spark_dtypes = [ (np.int8, [("CAST(1 AS TINYINT)", "tinyint")]), (np.int16, [("CAST(1 AS SMALLINT)", "smallint")]), (np.int32, [("CAST(1 AS INT)", "int")]), (np.int64, [("CAST(1 AS BIGINT)", "bigint")]), (np.float32, [("CAST(1.0 AS FLOAT)", "float")]), (np.float64, [("CAST(1.0 AS DOUBLE)", "double")]), (np.bool_, [("true", "boolean")]), ] for dtype, spark_dtypes in dtype_to_spark_dtypes: self.assertEqual(self.spark.range(1).select(lit(dtype(1))).dtypes, spark_dtypes)
Traceback (most recent call last): File "/Users/s.singh/personal/spark-oss/python/pyspark/sql/tests/test_functions.py", line 1064, in test_lit_np_scalar self.assertEqual(self.spark.range(1).select(lit(dtype(1))).dtypes, spark_dtypes) File "/Users/s.singh/personal/spark-oss/python/pyspark/sql/utils.py", line 332, in wrapped return getattr(functions, f.__name__)(*args, **kwargs) File "/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py", line 198, in lit return Column(LiteralExpression._from_value(col)) File "/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/expressions.py", line 266, in _from_value return LiteralExpression(value=value, dataType=LiteralExpression._infer_type(value)) File "/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/expressions.py", line 262, in _infer_type raise ValueError(f"Unsupported Data Type {type(value).__name__}") ValueError: Unsupported Data Type int8
