[SPARK-48084] pyspark.ml.connect.evaluation not working in 3.5 client <> 4.0 server

• Type: Sub-task
• Status: Resolved
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 4.0.0
• Fix Version/s: 3.5.2
• Component/s: ML, PySpark
• Assignee: Weichen Xu
• Reporter: Hyukjin Kwon
• Created: 02/May/24 09:34
• Updated: 07/May/24 06:58
• Resolved: 07/May/24 06:58

Description

======================================================================
ERROR [3.966s]: test_regressor_evaluator (pyspark.ml.tests.connect.test_connect_evaluation.EvaluationTestsOnConnect.test_regressor_evaluator)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py", line 69, in test_regressor_evaluator
    rmse = rmse_evaluator.evaluate(df1)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/base.py", line 255, in evaluate
    return self._evaluate(dataset)
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/evaluation.py", line 70, in _evaluate
    return aggregate_dataframe(
           ^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/util.py", line 93, in aggregate_dataframe
    state = cloudpickle.loads(state)
            ^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: Can't get attribute '_class_setstate' on <module 'pyspark.cloudpickle.cloudpickle' from '/home/runner/work/spark/spark-3.5/python/pyspark/cloudpickle/cloudpickle.py'>
----------------------------------------------------------------------

======================================================================
ERROR [4.664s]: test_copy (pyspark.ml.tests.connect.test_connect_tuning.CrossValidatorTestsOnConnect.test_copy)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py", line 115, in test_copy
    cvModel = cv.fit(dataset)
              ^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/base.py", line 106, in fit
    return self._fit(dataset)
           ^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/tuning.py", line 437, in _fit
    for j, metric in pool.imap_unordered(lambda f: f(), tasks):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/multiprocessing/pool.py", line 873, in next
    raise value
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/multiprocessing/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
                    ^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/tuning.py", line 437, in <lambda>
    for j, metric in pool.imap_unordered(lambda f: f(), tasks):
                                                   ^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/tuning.py", line 188, in single_task
    metric = evaluator.evaluate(
             ^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/base.py", line 255, in evaluate
    return self._evaluate(dataset)
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/evaluation.py", line 70, in _evaluate
    return aggregate_dataframe(
           ^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/util.py", line 93, in aggregate_dataframe
    state = cloudpickle.loads(state)
            ^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: Can't get attribute '_class_setstate' on <module 'pyspark.cloudpickle.cloudpickle' from '/home/runner/work/spark/spark-3.5/python/pyspark/cloudpickle/cloudpickle.py'>

======================================================================
ERROR [3.938s]: test_fit_minimize_metric (pyspark.ml.tests.connect.test_connect_tuning.CrossValidatorTestsOnConnect.test_fit_minimize_metric)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py", line 149, in test_fit_minimize_metric
    cvModel = cv.fit(dataset)
              ^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/base.py", line 106, in fit
    return self._fit(dataset)
           ^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/tuning.py", line 437, in _fit
    for j, metric in pool.imap_unordered(lambda f: f(), tasks):
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/multiprocessing/pool.py", line 873, in next
    raise value
  File "/opt/hostedtoolcache/Python/3.11.9/x64/lib/python3.11/multiprocessing/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
                    ^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/tuning.py", line 437, in <lambda>
    for j, metric in pool.imap_unordered(lambda f: f(), tasks):
                                                   ^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/tuning.py", line 188, in single_task
    metric = evaluator.evaluate(
             ^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/base.py", line 255, in evaluate
    return self._evaluate(dataset)
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/evaluation.py", line 70, in _evaluate
    return aggregate_dataframe(
           ^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/spark/spark-3.5/python/pyspark/ml/connect/util.py", line 93, in aggregate_dataframe
    state = cloudpickle.loads(state)
            ^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: Can't get attribute '_class_setstate' on <module 'pyspark.cloudpickle.cloudpickle' from '/home/runner/work/spark/spark-3.5/python/pyspark/cloudpickle/cloudpickle.py'>