[LIVY-370] livy-python-api build fail

• Type: Bug
• Status: Resolved
• Priority: Major
• Resolution: Not A Problem
• Affects Version/s: 0.4.0
• Component/s: Tests
• Labels: broken-build
• Assignee: Unassigned
• Reporter: Anirudh Jonnadula
• Created: 15/Jun/17 22:54
• Updated: 23/Jan/18 23:26
• Resolved: 23/Jan/18 23:26

Description

Building the master branch by invoking "mvn package" results in a BUILD FAILURE for livy-python-api
Below is the detailed error:
============================= test session starts ==============================
platform darwin – Python 2.7.10, pytest-3.1.2, py-1.4.34, pluggy-0.4.0 – /usr/bin/python
cachedir: .cache
rootdir: /Users/${USER}/dev/workspace/livy/python-api, inifile: setup.cfg
collecting ... collected 12 items

src/test/python/livy-tests/client_test.py::test_create_new_session_without_default_config FAILED
src/test/python/livy-tests/client_test.py::test_create_new_session_with_default_config FAILED
src/test/python/livy-tests/client_test.py::test_connect_to_existing_session FAILED
src/test/python/livy-tests/client_test.py::test_add_jar <- <string> FAILED
src/test/python/livy-tests/client_test.py::test_submit_job_verify_running_state <- <string> FAILED
src/test/python/livy-tests/client_test.py::test_submit_job_verify_failed_state <- <string> FAILED
src/test/python/livy-tests/client_test.py::test_add_pyfile <- <string> FAILED
src/test/python/livy-tests/client_test.py::test_submit_job_verify_queued_state <- <string> FAILED
src/test/python/livy-tests/client_test.py::test_add_file <- <string> FAILED
src/test/python/livy-tests/client_test.py::test_upload_file <- <string> FAILED
src/test/python/livy-tests/client_test.py::test_upload_pyfile <- <string> FAILED
src/test/python/livy-tests/client_test.py::test_submit_job_verify_succeeded_state <- <string> FAILED

=================================== FAILURES ===================================
________________ test_create_new_session_without_default_config ________________

def test_create_new_session_without_default_config():
> mock_and_validate_create_new_session(False)

src/test/python/livy-tests/client_test.py:104:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
<string>:3: in wrapper
???
src/test/python/livy-tests/client_test.py:47: in mock_and_validate_create_new_session
load_defaults=defaults)
src/main/python/livy/client.py:88: in _init_
session_conf_dict).json()['id']
src/main/python/livy/client.py:388: in _create_new_session
headers=self._conn._JSON_HEADERS, data=data)
src/main/python/livy/client.py:499: in send_request
json=data, auth=self._spnego_auth())
.eggs/requests-2.18.1-py2.7.egg/requests/api.py:58: in request
return session.request(method=method, url=url, **kwargs)
.eggs/requests-2.18.1-py2.7.egg/requests/sessions.py:502: in request
resp = self.send(prep, **send_kwargs)
.eggs/requests-2.18.1-py2.7.egg/requests/sessions.py:612: in send
r = adapter.send(request, **kwargs)
.eggs/responses-0.5.1-py2.7.egg/responses.py:294: in unbound_on_send
return self._on_request(adapter, request, *a, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <responses.RequestsMock object at 0x102c75790>
adapter = <requests.adapters.HTTPAdapter object at 0x102ea5bd0>
request = <PreparedRequest [POST]>
kwargs =

{'cert': None, 'proxies': OrderedDict(), 'stream': False, 'timeout': 10, ...}

match = None
error_msg = 'Connection refused: POST http://c02jw0tgdkq2.local:8998/sessions/'
response = ConnectionError(u'Connection refused: POST http://c02jw0tgdkq2.local:8998/sessions/',)

def _on_request(self, adapter, request, **kwargs):
match = self._find_match(request)

1. TODO(dcramer): find the correct class for this
   if match is None:
   error_msg = 'Connection refused: {0} {1}'.format(request.method,
   request.url)
   response = ConnectionError(error_msg)
   response.request = request
   
   self._calls.add(request, response)
   > raise response
   E ConnectionError: Connection refused: POST http://c02jw0tgdkq2.local:8998/sessions/
   
   .eggs/responses-0.5.1-py2.7.egg/responses.py:239: ConnectionError
   _________________ test_create_new_session_with_default_config __________________
   
   def test_create_new_session_with_default_config():
   os.environ["LIVY_CLIENT_CONF_DIR"] = \
   os.path.dirname(os.path.abspath(_file_)) + "/resources"
   > mock_and_validate_create_new_session(True)
   
   src/test/python/livy-tests/client_test.py:110:
   _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
   <string>:3: in wrapper
   ???
   src/test/python/livy-tests/client_test.py:47: in mock_and_validate_create_new_session
   load_defaults=defaults)
   src/main/python/livy/client.py:88: in _init_
   session_conf_dict).json()['id']
   src/main/python/livy/client.py:388: in _create_new_session
   headers=self._conn._JSON_HEADERS, data=data)
   src/main/python/livy/client.py:499: in send_request
   json=data, auth=self._spnego_auth())
   .eggs/requests-2.18.1-py2.7.egg/requests/api.py:58: in request
   return session.request(method=method, url=url, **kwargs)
   .eggs/requests-2.18.1-py2.7.egg/requests/sessions.py:502: in request
   resp = self.send(prep, **send_kwargs)
   .eggs/requests-2.18.1-py2.7.egg/requests/sessions.py:612: in send
   r = adapter.send(request, **kwargs)
   .eggs/responses-0.5.1-py2.7.egg/responses.py:294: in unbound_on_send
   return self._on_request(adapter, request, *a, **kwargs)
   _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
   
   self = <responses.RequestsMock object at 0x102c75790>
   adapter = <requests.adapters.HTTPAdapter object at 0x102fa46d0>
   request = <PreparedRequest [POST]>
   kwargs = {'cert': None, 'proxies': OrderedDict(), 'stream': False, 'timeout': 10, ...}
   match = None
   error_msg = 'Connection refused: POST http://c02jw0tgdkq2.local:8998/sessions/'
   response = ConnectionError(u'Connection refused: POST http://c02jw0tgdkq2.local:8998/sessions/',)
   
   def _on_request(self, adapter, request, **kwargs):
   match = self._find_match(request)
   # TODO(dcramer): find the correct class for this
   if match is None:
   error_msg = 'Connection refused: {0}
   {1}

   '.format(request.method,
   request.url)
   response = ConnectionError(error_msg)
   response.request = request

self._calls.add(request, response)
> raise response
E ConnectionError: Connection refused: POST http://c02jw0tgdkq2.local:8998/sessions/

.eggs/responses-0.5.1-py2.7.egg/responses.py:239: ConnectionError
_______________________ test_connect_to_existing_session _______________________

def test_connect_to_existing_session():
reconnect_mock_request_uri = base_uri + "/sessions/" + str(session_id) + \
"/connect"
reconnect_session_uri = base_uri + "/sessions/" + str(session_id)
json_data =

{ u'kind': u'pyspark', u'log': [], u'proxyUser': None, u'state': u'starting', u'owner': None, u'id': session_id }

with responses.RequestsMock() as rsps:
rsps.add(responses.POST, reconnect_mock_request_uri, json=json_data,
status=201, content_type='application/json')

client_reconnect = HttpClient(reconnect_session_uri,
> load_defaults=False)

src/test/python/livy-tests/client_test.py:126:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.eggs/responses-0.5.1-py2.7.egg/responses.py:181: in _exit_
self.stop()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <responses.RequestsMock object at 0x102f4a110>

def stop(self):
self._patcher.stop()
if self.assert_all_requests_are_fired and self._urls:
raise AssertionError(
'Not all requests have been executed

{0!r}

'.format(
> [(url['method'], url['url']) for url in self._urls]))
E AssertionError: Not all requests have been executed [(u'POST', 'http://C02JW0TGDKQ2.local:8998/sessions/0/connect')]

.eggs/responses-0.5.1-py2.7.egg/responses.py:304: AssertionError
_________________________________ test_add_jar _________________________________

> ???

<string>:3:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

@responses.activate
def test_add_jar():
file_uri = "file://" + os.path.dirname(os.path.abspath(_file_)) + \
"/resources/jar_file.jar"
> add_file_future = mock_file_apis('add-jar', client_test.add_jar, file_uri)
E AttributeError: 'NoneType' object has no attribute 'add_jar'

src/test/python/livy-tests/client_test.py:225: AttributeError
_____________________ test_submit_job_verify_running_state _____________________

> ???

<string>:3:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
src/test/python/livy-tests/client_test.py:137: in test_submit_job_verify_running_state
u'STARTED')
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

job = <function simple_spark_job at 0x102dbda28>, job_state = 'STARTED'
result = None, error = None

def mock_submit_job_and_poll_result(
job,
job_state,
result=None,
error=None
):
submit_request_mock_uri = base_uri + "/sessions/" + str(session_id) \
+ "/submit-job"
poll_request_mock_uri = base_uri + "/sessions/" + str(session_id) \
+ "/jobs/" + str(job_id)

post_json_data =

{ u'state': u'SENT', u'error': None, u'id': job_id, u'result': None }

responses.add(responses.POST, submit_request_mock_uri, status=201,
json=post_json_data, content_type='application/json')

get_json_data =

{ u'state': job_state, u'error': error, u'id': job_id, u'result': result }

responses.add(responses.GET, poll_request_mock_uri, status=200,
json=get_json_data, content_type='application/json')

> submit_job_future = client_test.submit(job)
E AttributeError: 'NoneType' object has no attribute 'submit'

src/test/python/livy-tests/client_test.py:80: AttributeError
_____________________ test_submit_job_verify_failed_state ______________________

> ???

<string>:3:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
src/test/python/livy-tests/client_test.py:176: in test_submit_job_verify_failed_state
error='Error job')
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

job = <function failure_job at 0x102dc00c8>, job_state = 'FAILED', result = None
error = 'Error job'

def mock_submit_job_and_poll_result(
job,
job_state,
result=None,
error=None
):
submit_request_mock_uri = base_uri + "/sessions/" + str(session_id) \
+ "/submit-job"
poll_request_mock_uri = base_uri + "/sessions/" + str(session_id) \
+ "/jobs/" + str(job_id)

post_json_data =

{ u'state': u'SENT', u'error': None, u'id': job_id, u'result': None }

responses.add(responses.POST, submit_request_mock_uri, status=201,
json=post_json_data, content_type='application/json')

get_json_data =

{ u'state': job_state, u'error': error, u'id': job_id, u'result': result }

responses.add(responses.GET, poll_request_mock_uri, status=200,
json=get_json_data, content_type='application/json')

> submit_job_future = client_test.submit(job)
E AttributeError: 'NoneType' object has no attribute 'submit'

src/test/python/livy-tests/client_test.py:80: AttributeError
_______________________________ test_add_pyfile ________________________________

> ???

<string>:3:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

@responses.activate
def test_add_pyfile():
file_uri = "file://" + os.path.dirname(os.path.abspath(_file_)) + \
"/resources/zip_file.zip"
> add_file_future = mock_file_apis('add-pyfile', client_test.add_pyfile,
file_uri)
E AttributeError: 'NoneType' object has no attribute 'add_pyfile'

src/test/python/livy-tests/client_test.py:205: AttributeError
_____________________ test_submit_job_verify_queued_state ______________________

> ???

<string>:3:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
src/test/python/livy-tests/client_test.py:152: in test_submit_job_verify_queued_state
u'QUEUED')
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

job = <function simple_spark_job at 0x102dbda28>, job_state = 'QUEUED'
result = None, error = None

def mock_submit_job_and_poll_result(
job,
job_state,
result=None,
error=None
):
submit_request_mock_uri = base_uri + "/sessions/" + str(session_id) \
+ "/submit-job"
poll_request_mock_uri = base_uri + "/sessions/" + str(session_id) \
+ "/jobs/" + str(job_id)

post_json_data =

{ u'state': u'SENT', u'error': None, u'id': job_id, u'result': None }

responses.add(responses.POST, submit_request_mock_uri, status=201,
json=post_json_data, content_type='application/json')

get_json_data =

{ u'state': job_state, u'error': error, u'id': job_id, u'result': result }

responses.add(responses.GET, poll_request_mock_uri, status=200,
json=get_json_data, content_type='application/json')

> submit_job_future = client_test.submit(job)
E AttributeError: 'NoneType' object has no attribute 'submit'

src/test/python/livy-tests/client_test.py:80: AttributeError
________________________________ test_add_file _________________________________

> ???

<string>:3:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

@responses.activate
def test_add_file():
file_uri = "file://" + os.path.dirname(os.path.abspath(_file_)) + \
"/resources/text_file.txt"
> add_file_future = mock_file_apis('add-file', client_test.add_file,
file_uri)
E AttributeError: 'NoneType' object has no attribute 'add_file'

src/test/python/livy-tests/client_test.py:185: AttributeError
_______________________________ test_upload_file _______________________________

> ???

<string>:3:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

@responses.activate
def test_upload_file():
file_path = os.path.dirname(os.path.abspath(_file_)) + \
"/resources/text_file.txt"
> upload_file_future = mock_file_apis('upload-file', client_test.upload_file,
file_path)
E AttributeError: 'NoneType' object has no attribute 'upload_file'

src/test/python/livy-tests/client_test.py:195: AttributeError
______________________________ test_upload_pyfile ______________________________

> ???

<string>:3:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

@responses.activate
def test_upload_pyfile():
file_path = os.path.dirname(os.path.abspath(_file_)) + \
"/resources/zip_file.zip"
> pyfile_future = mock_file_apis('upload-pyfile', client_test.upload_pyfile,
file_path)
E AttributeError: 'NoneType' object has no attribute 'upload_pyfile'

src/test/python/livy-tests/client_test.py:215: AttributeError
____________________ test_submit_job_verify_succeeded_state ____________________

> ???

<string>:3:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
src/test/python/livy-tests/client_test.py:168: in test_submit_job_verify_succeeded_state
result='Z0FKVkZGc3hNREFzSURJd01Dd2dNekF3TENBME1EQmRjUUF1')
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

job = <function simple_spark_job at 0x102dbda28>, job_state = 'SUCCEEDED'
result = 'Z0FKVkZGc3hNREFzSURJd01Dd2dNekF3TENBME1EQmRjUUF1', error = None

def mock_submit_job_and_poll_result(
job,
job_state,
result=None,
error=None
):
submit_request_mock_uri = base_uri + "/sessions/" + str(session_id) \
+ "/submit-job"
poll_request_mock_uri = base_uri + "/sessions/" + str(session_id) \
+ "/jobs/" + str(job_id)

post_json_data =

{ u'state': u'SENT', u'error': None, u'id': job_id, u'result': None }

responses.add(responses.POST, submit_request_mock_uri, status=201,
json=post_json_data, content_type='application/json')

get_json_data =

{ u'state': job_state, u'error': error, u'id': job_id, u'result': result }

responses.add(responses.GET, poll_request_mock_uri, status=200,
json=get_json_data, content_type='application/json')

> submit_job_future = client_test.submit(job)
E AttributeError: 'NoneType' object has no attribute 'submit'

src/test/python/livy-tests/client_test.py:80: AttributeError
========================== 12 failed in 0.76 seconds ===========================
[INFO] ------------------------------------------------------------------------
[INFO] BUILD FAILURE
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 4.908 s
[INFO] Finished at: 2017-06-15T13:59:36-07:00
[INFO] Final Memory: 24M/299M
[INFO] ------------------------------------------------------------------------
[ERROR] Failed to execute goal org.codehaus.mojo:exec-maven-plugin:1.2.1:exec (python-api test) on project livy-python-api: Command execution failed.: Process exited with an error: 1 (Exit value: 1) -> [Help 1]
[ERROR]
[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch.
[ERROR] Re-run Maven using the -X switch to enable full debug logging.
[ERROR]
[ERROR] For more information about the errors and possible solutions, please read the following articles:
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException