(spark_dev) ➜ spark git:(master) bin/pyspark --remote "local[*]"
Python 3.9.15 (main, Nov 24 2022, 08:28:41)
Type 'copyright', 'credits' or 'license' for more information
IPython 8.9.0 -- An enhanced Interactive Python. Type '?' for help.
/Users/ruifeng.zheng/Dev/spark/python/pyspark/shell.py:45: UserWarning: Failed to initialize Spark session.
warnings.warn("Failed to initialize Spark session.")
Traceback (most recent call last):
File "/Users/ruifeng.zheng/Dev/spark/python/pyspark/shell.py", line 40, in <module>
spark = SparkSession.builder.getOrCreate()
File "/Users/ruifeng.zheng/Dev/spark/python/pyspark/sql/session.py", line 429, in getOrCreate
from pyspark.sql.connect.session import SparkSession as RemoteSparkSession
File "/Users/ruifeng.zheng/Dev/spark/python/pyspark/sql/connect/__init__.py", line 21, in <module>
from pyspark.sql.connect.dataframe import DataFrame # noqa: F401
File "/Users/ruifeng.zheng/Dev/spark/python/pyspark/sql/connect/dataframe.py", line 35, in <module>
import pandas
File "/Users/ruifeng.zheng/Dev/spark/python/pyspark/pandas/__init__.py", line 29, in <module>
from pyspark.pandas.missing.general_functions import MissingPandasLikeGeneralFunctions
File "/Users/ruifeng.zheng/Dev/spark/python/pyspark/pandas/__init__.py", line 34, in <module>
require_minimum_pandas_version()
File "/Users/ruifeng.zheng/Dev/spark/python/pyspark/sql/pandas/utils.py", line 37, in require_minimum_pandas_version
if LooseVersion(pandas.__version__) < LooseVersion(minimum_pandas_version):
AttributeError: partially initialized module 'pandas' has no attribute '__version__' (most likely due to a circular import)
[TerminalIPythonApp] WARNING | Unknown error in handling PYTHONSTARTUP file /Users/ruifeng.zheng/Dev/spark---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
File ~/Dev/spark/python/pyspark/shell.py:40
38 try:
39 # Creates pyspark.sql.connect.SparkSession.
---> 40 spark = SparkSession.builder.getOrCreate()
41 except Exception:
File ~/Dev/spark/python/pyspark/sql/session.py:429, in SparkSession.Builder.getOrCreate(self)
428 with SparkContext._lock:
--> 429 from pyspark.sql.connect.session import SparkSession as RemoteSparkSession
431 if (
432 SparkContext._active_spark_context is None
433 and SparkSession._instantiatedSession is None
434 ):
File ~/Dev/spark/python/pyspark/sql/connect/__init__.py:21
18 """Currently Spark Connect is very experimental and the APIs to interact with
19 Spark through this API are can be changed at any time without warning."""
---> 21 from pyspark.sql.connect.dataframe import DataFrame # noqa: F401
22 from pyspark.sql.pandas.utils import (
23 require_minimum_pandas_version,
24 require_minimum_pyarrow_version,
25 require_minimum_grpc_version,
26 )
File ~/Dev/spark/python/pyspark/sql/connect/dataframe.py:35
34 import random
---> 35 import pandas
36 import json
File ~/Dev/spark/python/pyspark/pandas/__init__.py:29
27 from typing import Any
---> 29 from pyspark.pandas.missing.general_functions import MissingPandasLikeGeneralFunctions
30 from pyspark.pandas.missing.scalars import MissingPandasLikeScalars
File ~/Dev/spark/python/pyspark/pandas/__init__.py:34
33 try:
---> 34 require_minimum_pandas_version()
35 require_minimum_pyarrow_version()
File ~/Dev/spark/python/pyspark/sql/pandas/utils.py:37, in require_minimum_pandas_version()
34 raise ImportError(
35 "Pandas >= %s must be installed; however, " "it was not found." % minimum_pandas_version
36 ) from raised_error
---> 37 if LooseVersion(pandas.__version__) < LooseVersion(minimum_pandas_version):
38 raise ImportError(
39 "Pandas >= %s must be installed; however, "
40 "your version was %s." % (minimum_pandas_version, pandas.__version__)
41 )
AttributeError: partially initialized module 'pandas' has no attribute '__version__' (most likely due to a circular import)
During handling of the above exception, another exception occurred:
SystemExit Traceback (most recent call last)
File ~/.dev/miniconda3/envs/spark_dev/lib/python3.9/site-packages/IPython/core/shellapp.py:359, in InteractiveShellApp._exec_file(self, fname, shell_futures)
355 self.shell.safe_execfile_ipy(full_filename,
356 shell_futures=shell_futures)
357 else:
358 # default to python, even without extension
--> 359 self.shell.safe_execfile(full_filename,
360 self.shell.user_ns,
361 shell_futures=shell_futures,
362 raise_exceptions=True)
363 finally:
364 sys.argv = save_argv
File ~/.dev/miniconda3/envs/spark_dev/lib/python3.9/site-packages/IPython/core/interactiveshell.py:2802, in InteractiveShell.safe_execfile(self, fname, exit_ignore, raise_exceptions, shell_futures, *where)
2800 try:
2801 glob, loc = (where + (None, ))[:2]
-> 2802 py3compat.execfile(
2803 fname, glob, loc,
2804 self.compile if shell_futures else None)
2805 except SystemExit as status:
2806 # If the call was made with 0 or None exit status (sys.exit(0)
2807 # or sys.exit() ), don't bother showing a traceback, as both of
(...)
2813 # For other exit status, we show the exception unless
2814 # explicitly silenced, but only in short form.
2815 if status.code:
File ~/.dev/miniconda3/envs/spark_dev/lib/python3.9/site-packages/IPython/utils/py3compat.py:55, in execfile(fname, glob, loc, compiler)
53 with open(fname, "rb") as f:
54 compiler = compiler or compile
---> 55 exec(compiler(f.read(), fname, "exec"), glob, loc)
File ~/Dev/spark/python/pyspark/shell.py:47
45 warnings.warn("Failed to initialize Spark session.")
46 traceback.print_exc(file=sys.stderr)
---> 47 sys.exit(1)
48 version = pyspark.__version__
49 sc = None
SystemExit: 1
Major 