[ARROW-10934] [Python] Tests are failed with fsspec-0.8.5

• Type: Improvement
• Status: Resolved
• Priority: Blocker
• Resolution: Fixed
• Fix Version/s: 3.0.0
• Component/s: Python
• Labels: pull-request-available
• External issue URL: https://github.com/apache/arrow/issues/26861
• Assignee: Joris Van den Bossche
• Reporter: Kouhei Sutou
• Created: 16/Dec/20 03:23
• Updated: 11/Jan/23 08:16
• Resolved: 16/Dec/20 10:08

Description

https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/36851219/job/lwywl76d82coawpd?fullLog=true#L2284

================================== FAILURES ===================================
_ test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))] _
fs = <pyarrow._fs.PyFileSystem object at 0x00000140F4C0A3A0>
pathfn = <function py_fsspec_memoryfs.<locals>.<lambda> at 0x00000140F4BFBB58>
    def test_get_file_info_with_selector(fs, pathfn):
        base_dir = pathfn('selector-dir/')
        file_a = pathfn('selector-dir/test_file_a')
        file_b = pathfn('selector-dir/test_file_b')
        dir_a = pathfn('selector-dir/test_dir_a')
        file_c = pathfn('selector-dir/test_dir_a/test_file_c')
        dir_b = pathfn('selector-dir/test_dir_b')
    
        try:
            fs.create_dir(base_dir)
            with fs.open_output_stream(file_a):
                pass
            with fs.open_output_stream(file_b):
                pass
            fs.create_dir(dir_a)
            with fs.open_output_stream(file_c):
                pass
            fs.create_dir(dir_b)
    
            # recursive selector
            selector = FileSelector(base_dir, allow_not_found=False,
                                    recursive=True)
            assert selector.base_dir == base_dir
    
            infos = fs.get_file_info(selector)
            if fs.type_name == "py::fsspec+s3":
                # s3fs only lists directories if they are not empty
                assert len(infos) == 4
            else:
                assert len(infos) == 5
    
            for info in infos:
                if (info.path.endswith(file_a) or info.path.endswith(file_b) or
                        info.path.endswith(file_c)):
                    assert info.type == FileType.File
                elif (info.path.rstrip("/").endswith(dir_a) or
                      info.path.rstrip("/").endswith(dir_b)):
                    assert info.type == FileType.Directory
                else:
                    raise ValueError('unexpected path {}'.format(info.path))
                check_mtime_or_absent(info)
    
            # non-recursive selector -> not selecting the nested file_c
            selector = FileSelector(base_dir, recursive=False)
    
            infos = fs.get_file_info(selector)
            if fs.type_name == "py::fsspec+s3":
                # s3fs only lists directories if they are not empty
                assert len(infos) == 3
            else:
                assert len(infos) == 4
    
        finally:
>           fs.delete_dir(base_dir)
pyarrow\tests\test_fs.py:716: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
pyarrow\_fs.pyx:472: in pyarrow._fs.FileSystem.delete_dir
    check_status(self.fs.DeleteDir(directory))
pyarrow\_fs.pyx:1035: in pyarrow._fs._cb_delete_dir
    handler.delete_dir(frombytes(path))
pyarrow\fs.py:262: in delete_dir
    self.fs.rm(path, recursive=True)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:176: in rm
    self.rm_file(p)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\spec.py:840: in rm_file
    self._rm(path)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:163: in _rm
    self.rmdir(path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <fsspec.implementations.memory.MemoryFileSystem object at 0x00000140F4ABBC58>
path = 'selector-dir'
    def rmdir(self, path):
        path = path.rstrip("/")
        if path in self.pseudo_dirs:
            if not self.ls(path):
                self.pseudo_dirs.remove(path)
            else:
>               raise OSError(ENOTEMPTY, "Directory not empty", path)
E               OSError: [Errno 41] Directory not empty: 'selector-dir'
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:110: OSError
__ test_delete_dir[PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))] __
fs = <pyarrow._fs.PyFileSystem object at 0x00000140F4C1CDC0>
pathfn = <function py_fsspec_memoryfs.<locals>.<lambda> at 0x00000140F50BC738>
    def test_delete_dir(fs, pathfn):
        skip_fsspec_s3fs(fs)
    
        d = pathfn('directory/')
        nd = pathfn('directory/nested/')
    
        fs.create_dir(nd)
>       fs.delete_dir(d)
pyarrow\tests\test_fs.py:743: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
pyarrow\_fs.pyx:472: in pyarrow._fs.FileSystem.delete_dir
    check_status(self.fs.DeleteDir(directory))
pyarrow\_fs.pyx:1035: in pyarrow._fs._cb_delete_dir
    handler.delete_dir(frombytes(path))
pyarrow\fs.py:262: in delete_dir
    self.fs.rm(path, recursive=True)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:176: in rm
    self.rm_file(p)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\spec.py:840: in rm_file
    self._rm(path)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:163: in _rm
    self.rmdir(path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <fsspec.implementations.memory.MemoryFileSystem object at 0x00000140F4ABBC58>
path = 'directory'
    def rmdir(self, path):
        path = path.rstrip("/")
        if path in self.pseudo_dirs:
            if not self.ls(path):
                self.pseudo_dirs.remove(path)
            else:
>               raise OSError(ENOTEMPTY, "Directory not empty", path)
E               OSError: [Errno 41] Directory not empty: 'directory'
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:110: OSError

Issue Links

• links to: GitHub Pull Request #8935