Details
-
Bug
-
Status: Resolved
-
Major
-
Resolution: Fixed
-
SystemDS 2.2, SystemDS 3.1
-
None
Description
On pyspark==3.2.1 and systemds==2.2.1 I'm trying to migrate the following SystemML DSL to SystemDS pythonDSL:
dml_script = '''
PI = 3.141592654
N = nrow(signal)
n = seq(0, N-1, 1)
k = seq(0, N-1, 1)
M = (n %% t(k))(2*PI/N)
Xa = cos(M) %*% signal
Xb = sin(M) %*% signal
DFT = cbind(Xa, Xb)
'''
from pyspark.sql.functions import monotonically_increasing_id
from systemds.context import SystemDSContext
import numpy as np
def dft_systemds(signal,name):
with SystemDSContext(spark) as sds:
pi = sds.scalar(3.141592654)
size = len(signal)
n = sds.from_numpy(np.array(range(0, size-1, 1)))
k = sds.from_numpy(np.array(range(0, size-1, 1)))
M = n._matmul_(k.t())*(2*pi/size)
Xa = M.cos().t()._matmul_(sds.from_numpy(np.array(signal)))
#Xb = M.cos()._matmul_(sds.from_numpy(np.array(signal)))
Xa_c = Xa.compute()
#Xb_c = Xb.compute()
#DFT = cbind(Xa, Xb)
dft_systemds([0,1,2],'x')
I get:
--------------------------------------------------------------------------- RuntimeError Traceback (most recent call last) /tmp/wsuser/ipykernel_122/2034152834.py in <module> 28 29 ---> 30 dft_systemml([0,1,2],'x') /tmp/wsuser/ipykernel_122/2034152834.py in dft_systemml(signal, name) 18 Xa = M.cos().t()._matmul(sds.from_numpy(np.array(signal))) 19 #Xb = M.cos().matmul_(sds.from_numpy(np.array(signal))) ---> 20 Xa_c = Xa.compute() 21 #Xb_c = Xb.compute() 22 /opt/conda/envs/Python-3.9/lib/python3.9/site-packages/systemds/operator/nodes/matrix.py in compute(self, verbose, lineage) 75 return self._np_array 76 else: ---> 77 return super().compute(verbose, lineage) 78 79 def _parse_output_result_variables(self, result_variables): /opt/conda/envs/Python-3.9/lib/python3.9/site-packages/systemds/operator/operation_node.py in compute(self, verbose, lineage) 99 result_variables, self._lineage_trace = self._script.execute_with_lineage() 100 else: --> 101 result_variables = self._script.execute() 102 103 if result_variables is not None: /opt/conda/envs/Python-3.9/lib/python3.9/site-packages/systemds/script_building/script.py in execute(self) 87 exception_str = str(e) 88 trace_back_limit = None ---> 89 self.sds_context.exception_and_close(exception_str, trace_back_limit) 90 91 /opt/conda/envs/Python-3.9/lib/python3.9/site-packages/systemds/context/systemds_context.py in exception_and_close(self, exception, trace_back_limit) 107 sys.tracebacklimit = trace_back_limit 108 self.close() --> 109 raise RuntimeError(message) 110 111 def __try_startup(self, command) -> bool: RuntimeError: standard error : WARNING: An illegal reflective access operation has occurred WARNING: Illegal reflective access by org.apache.hadoop.security.authentication.util.KerberosUtil (file:/opt/conda/envs/Python-3.9/lib/python3.9/site-packages/systemds/lib/hadoop-auth-2.6.0.jar) to method sun.security.krb5.Config.getInstance() WARNING: Please consider reporting this to the maintainers of org.apache.hadoop.security.authentication.util.KerberosUtil WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations WARNING: All illegal access operations will be denied in a future release An error occurred while calling z:org.apache.spark.util.Utils.exceptionString. Trace: py4j.Py4JException: Method exceptionString([class org.apache.spark.SparkConf]) does not exist at py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:318) at py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:339) at py4j.Gateway.invoke(Gateway.java:276) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) at py4j.ClientServerConnection.run(ClientServerConnection.java:106) at java.base/java.lang.Thread.run(Thread.java:829)
Attachments
Issue Links
- links to