Details
-
Bug
-
Status: Closed
-
Blocker
-
Resolution: Cannot Reproduce
-
None
-
None
-
None
Description
Nightly-packaging jobs for wheels on MAC OS have started failing with PermissionError:
Build example:
https://github.com/ursacomputing/crossbow/runs/6074813901?check_suite_focus=true
The error is related with the following S3 test
==================================== ERRORS ==================================== ________ ERROR at setup of test_s3fs_limited_permissions_create_bucket _________request = <SubRequest 'limited_s3_user' for <Function test_s3fs_limited_permissions_create_bucket>> s3_server = {'connection': ('localhost', 57493, 'arrow', 'apachearrow'), 'process': <Popen: returncode: None args: ['minio', '--compat', 'server', '--quiet', '-...>, 'tempdir': '/var/folders/cn/d0vdy6t94_77fwckh55bz2300000gq/T/tmpor9puihr'} @pytest.fixture(scope='session') def limited_s3_user(request, s3_server): if sys.platform == 'win32': # Can't rely on FileNotFound check because # there is sometimes an mc command on Windows # which is unrelated to the minio mc pytest.skip('The mc command is not installed on Windows') request.config.pyarrow.requires('s3') tempdir = s3_server['tempdir'] host, port, access_key, secret_key = s3_server['connection'] address = '{}:{}'.format(host, port) > if not _configure_limited_user(tempdir, address, access_key, secret_key):test-arm64-env/lib/python3.9/site-packages/pyarrow/tests/test_fs.py:358: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test-arm64-env/lib/python3.9/site-packages/pyarrow/tests/test_fs.py:317: in _configure_limited_user if not _ensure_minio_component_version('mc', 2021): test-arm64-env/lib/python3.9/site-packages/pyarrow/tests/test_fs.py:292: in _ensure_minio_component_version proc = subprocess.Popen(full_args, stdout=subprocess.PIPE, /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/subprocess.py:951: in __init__ self._execute_child(args, executable, preexec_fn, close_fds, _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Popen: returncode: 255 args: ['mc', '--version']> args = ['mc', '--version'], executable = b'mc', preexec_fn = None close_fds = True, pass_fds = (), cwd = None, env = None, startupinfo = None creationflags = 0, shell = False, p2cread = -1, p2cwrite = -1, c2pread = 12 c2pwrite = 13, errread = 14, errwrite = 15, restore_signals = True, gid = None gids = None, uid = None, umask = -1, start_new_session = False def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, gid, gids, uid, umask, start_new_session): """Execute program (POSIX version)""" if isinstance(args, (str, bytes)): args = [args] elif isinstance(args, os.PathLike): if shell: raise TypeError('path-like args is not allowed when ' 'shell is true') args = [args] else: args = list(args) if shell: # On Android the default shell is at '/system/bin/sh'. unix_shell = ('/system/bin/sh' if hasattr(sys, 'getandroidapilevel') else '/bin/sh') args = [unix_shell, "-c"] + args if executable: args[0] = executable if executable is None: executable = args[0] sys.audit("subprocess.Popen", executable, args, cwd, env) if (_USE_POSIX_SPAWN and os.path.dirname(executable) and preexec_fn is None and not close_fds and not pass_fds and cwd is None and (p2cread == -1 or p2cread > 2) and (c2pwrite == -1 or c2pwrite > 2) and (errwrite == -1 or errwrite > 2) and not start_new_session and gid is None and gids is None and uid is None and umask < 0): self._posix_spawn(args, executable, env, restore_signals, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) return orig_executable = executable # For transferring possible exec failure from child to parent. # Data format: "exception name:hex errno:description" # Pickle is not used; it is complex and involves memory allocation. errpipe_read, errpipe_write = os.pipe() # errpipe_write must not be in the standard io 0, 1, or 2 fd range. low_fds_to_close = [] while errpipe_write < 3: low_fds_to_close.append(errpipe_write) errpipe_write = os.dup(errpipe_write) for low_fd in low_fds_to_close: os.close(low_fd) try: try: # We must avoid complex work that could involve # malloc or free in the child process to avoid # potential deadlocks, thus we do all this here. # and pass it to fork_exec() if env is not None: env_list = [] for k, v in env.items(): k = os.fsencode(k) if b'=' in k: raise ValueError("illegal environment variable name") env_list.append(k + b'=' + os.fsencode(v)) else: env_list = None # Use execv instead of execve. executable = os.fsencode(executable) if os.path.dirname(executable): executable_list = (executable,) else: # This matches the behavior of os._execvpe(). executable_list = tuple( os.path.join(os.fsencode(dir), executable) for dir in os.get_exec_path(env)) fds_to_keep = set(pass_fds) fds_to_keep.add(errpipe_write) self.pid = _posixsubprocess.fork_exec( args, executable_list, close_fds, tuple(sorted(map(int, fds_to_keep))), cwd, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, start_new_session, gid, gids, uid, umask, preexec_fn) self._child_created = True finally: # be sure the FD is closed no matter what os.close(errpipe_write) self._close_pipe_fds(p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) # Wait for exec to fail or succeed; possibly raising an # exception (limited in size) errpipe_data = bytearray() while True: part = os.read(errpipe_read, 50000) errpipe_data += part if not part or len(errpipe_data) > 50000: break finally: # be sure the FD is closed no matter what os.close(errpipe_read) if errpipe_data: try: pid, sts = os.waitpid(self.pid, 0) if pid == self.pid: self._handle_exitstatus(sts) else: self.returncode = sys.maxsize except ChildProcessError: pass try: exception_name, hex_errno, err_msg = ( errpipe_data.split(b':', 2)) # The encoding here should match the encoding # written in by the subprocess implementations # like _posixsubprocess err_msg = err_msg.decode() except ValueError: exception_name = b'SubprocessError' hex_errno = b'0' err_msg = 'Bad exception data from child: {!r}'.format( bytes(errpipe_data)) child_exception_type = getattr( builtins, exception_name.decode('ascii'), SubprocessError) if issubclass(child_exception_type, OSError) and hex_errno: errno_num = int(hex_errno, 16) child_exec_never_called = (err_msg == "noexec") if child_exec_never_called: err_msg = "" # The error must be from chdir(cwd). err_filename = cwd else: err_filename = orig_executable if errno_num != 0: err_msg = os.strerror(errno_num) > raise child_exception_type(errno_num, err_msg, err_filename) E PermissionError: [Errno 13] Permission denied: 'mc'
All the failed tasks with this error:
Failed Tasks:
- wheel-macos-big-sur-cp310-arm64:
URL: https://github.com/ursacomputing/crossbow/tree/nightly-packaging-2022-04-19-0-github-wheel-macos-big-sur-cp310-arm64 - wheel-macos-big-sur-cp310-universal2:
URL: https://github.com/ursacomputing/crossbow/tree/nightly-packaging-2022-04-19-0-github-wheel-macos-big-sur-cp310-universal2 - wheel-macos-big-sur-cp38-arm64:
URL: https://github.com/ursacomputing/crossbow/tree/nightly-packaging-2022-04-19-0-github-wheel-macos-big-sur-cp38-arm64 - wheel-macos-big-sur-cp39-arm64:
URL: https://github.com/ursacomputing/crossbow/tree/nightly-packaging-2022-04-19-0-github-wheel-macos-big-sur-cp39-arm64