[BEAM-11209] Python Unit Tests (windows-latest, 3.7, py37)

• Type: Bug
• Status: Resolved
• Priority: P1
• Resolution: Fixed
• Fix Version/s: 2.26.0
• Component/s: test-failures
• Assignee: Unassigned
• Reporter: Robert Burke
• Created: 09/Nov/20 19:54
• Updated: 15/May/21 17:59
• Resolved: 15/May/21 17:59

Description

Cut branch is failing python tests on windows.

https://github.com/apache/beam/pull/13275/checks?check_run_id=1361068184

{{================================== FAILURES ===================================
__________________ ElementStreamTest.test_done_if_terminated __________________
[gw1] win32 – Python 3.7.9 d:\a\beam\beam\sdks\python\target\.tox\py37-win\scripts\python.exe

self = <apache_beam.runners.interactive.recording_manager_test.ElementStreamTest testMethod=test_done_if_terminated>

def setUp(self):
> ie.new_env()

apache_beam\runners\interactive\recording_manager_test.py:75:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
apache_beam\runners\interactive\interactive_environment.py:118: in new_env
_interactive_beam_env.cleanup()
apache_beam\runners\interactive\interactive_environment.py:272: in cleanup
cache_manager.cleanup()
apache_beam\runners\interactive\caching\streaming_cache.py:391: in cleanup
shutil.rmtree(self._cache_dir)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:516: in rmtree
return _rmtree_unsafe(path, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:395: in _rmtree_unsafe
_rmtree_unsafe(fullname, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:400: in _rmtree_unsafe
onerror(os.unlink, fullname, sys.exc_info())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

path = 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-o3earxzq2327948941064
full'
onerror = <function rmtree.<locals>.onerror at 0x0000021E06D6C3A8>

def _rmtree_unsafe(path, onerror):
try:
with os.scandir(path) as scandir_it:
entries = list(scandir_it)
except OSError:
onerror(os.scandir, path, sys.exc_info())
entries = []
for entry in entries:
fullname = entry.path
try:
is_dir = entry.is_dir(follow_symlinks=False)
except OSError:
is_dir = False
if is_dir:
try:
if entry.is_symlink():

1. This can only happen if someone replaces
2. a directory with a symlink after the call to
3. os.scandir or entry.is_dir above.
   raise OSError("Cannot call rmtree on a symbolic link")
   except OSError:
   onerror(os.path.islink, fullname, sys.exc_info())
   continue
   _rmtree_unsafe(fullname, onerror)
   else:
   try:
   > os.unlink(fullname)
   E PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-o3earxzq2327948941064\\full
   6b31141af3-2327948942920-2327944863176-2327948941064'

c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:398: PermissionError
_________________________ ElementStreamTest.test_read _________________________
[gw1] win32 – Python 3.7.9 d:\a\beam\beam\sdks\python\target\.tox\py37-win\scripts\python.exe

self = <apache_beam.runners.interactive.recording_manager_test.ElementStreamTest testMethod=test_read>

def setUp(self):
> ie.new_env()

apache_beam\runners\interactive\recording_manager_test.py:75:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
apache_beam\runners\interactive\interactive_environment.py:118: in new_env
_interactive_beam_env.cleanup()
apache_beam\runners\interactive\interactive_environment.py:272: in cleanup
cache_manager.cleanup()
apache_beam\runners\interactive\caching\streaming_cache.py:391: in cleanup
shutil.rmtree(self._cache_dir)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:516: in rmtree
return _rmtree_unsafe(path, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:395: in _rmtree_unsafe
_rmtree_unsafe(fullname, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:400: in _rmtree_unsafe
onerror(os.unlink, fullname, sys.exc_info())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

path = 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-o3earxzq2327948941064
full'
onerror = <function rmtree.<locals>.onerror at 0x0000021E06E450D8>

def _rmtree_unsafe(path, onerror):
try:
with os.scandir(path) as scandir_it:
entries = list(scandir_it)
except OSError:
onerror(os.scandir, path, sys.exc_info())
entries = []
for entry in entries:
fullname = entry.path
try:
is_dir = entry.is_dir(follow_symlinks=False)
except OSError:
is_dir = False
if is_dir:
try:
if entry.is_symlink():

1. This can only happen if someone replaces
2. a directory with a symlink after the call to
3. os.scandir or entry.is_dir above.
   raise OSError("Cannot call rmtree on a symbolic link")
   except OSError:
   onerror(os.path.islink, fullname, sys.exc_info())
   continue
   _rmtree_unsafe(fullname, onerror)
   else:
   try:
   > os.unlink(fullname)
   E PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-o3earxzq2327948941064\\full
   6b31141af3-2327948942920-2327944863176-2327948941064'

c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:398: PermissionError}}
 

Issue Links

• duplicates: BEAM-10921 Interactive Runner Python unit tests are flaky on Windows