From 1cd4e45777188da86888ad1d40319cfbc969b6eb Mon Sep 17 00:00:00 2001 From: Moreal Date: Sun, 14 Apr 2024 18:15:31 +0900 Subject: [PATCH 1/3] Bump shutil to 3.12.3 --- Lib/shutil.py | 607 +++++++---- Lib/test/test_shutil.py | 2264 +++++++++++++++++++++++++-------------- vm/src/stdlib/os.rs | 15 + 3 files changed, 1871 insertions(+), 1015 deletions(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index 31336e08e8..6803ee3ce6 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -10,6 +10,7 @@ import fnmatch import collections import errno +import warnings try: import zlib @@ -32,16 +33,6 @@ except ImportError: _LZMA_SUPPORTED = False -try: - from pwd import getpwnam -except ImportError: - getpwnam = None - -try: - from grp import getgrnam -except ImportError: - getgrnam = None - _WINDOWS = os.name == 'nt' posix = nt = None if os.name == 'posix': @@ -49,10 +40,20 @@ elif _WINDOWS: import nt +if sys.platform == 'win32': + import _winapi +else: + _winapi = None + COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 +# This should never be removed, see rationale in: +# https://bugs.python.org/issue43743#msg393429 _USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS +# CMD defaults in Windows 10 +_WIN_DEFAULT_PATHEXT = ".COM;.EXE;.BAT;.CMD;.VBS;.JS;.WS;.MSC" + __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "copytree", "move", "rmtree", "Error", "SpecialFileError", "ExecError", "make_archive", "get_archive_formats", @@ -189,21 +190,19 @@ def _copyfileobj_readinto(fsrc, fdst, length=COPY_BUFSIZE): break elif n < length: with mv[:n] as smv: - fdst.write(smv) + fdst_write(smv) + break else: fdst_write(mv) def copyfileobj(fsrc, fdst, length=0): """copy data from file-like object fsrc to file-like object fdst""" - # Localize variable access to minimize overhead. if not length: length = COPY_BUFSIZE + # Localize variable access to minimize overhead. fsrc_read = fsrc.read fdst_write = fdst.write - while True: - buf = fsrc_read(length) - if not buf: - break + while buf := fsrc_read(length): fdst_write(buf) def _samefile(src, dst): @@ -260,28 +259,37 @@ def copyfile(src, dst, *, follow_symlinks=True): if not follow_symlinks and _islink(src): os.symlink(os.readlink(src), dst) else: - with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: - # macOS - if _HAS_FCOPYFILE: - try: - _fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA) - return dst - except _GiveupOnFastCopy: - pass - # Linux - elif _USE_CP_SENDFILE: - try: - _fastcopy_sendfile(fsrc, fdst) - return dst - except _GiveupOnFastCopy: - pass - # Windows, see: - # https://github.com/python/cpython/pull/7160#discussion_r195405230 - elif _WINDOWS and file_size > 0: - _copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE)) - return dst - - copyfileobj(fsrc, fdst) + with open(src, 'rb') as fsrc: + try: + with open(dst, 'wb') as fdst: + # macOS + if _HAS_FCOPYFILE: + try: + _fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA) + return dst + except _GiveupOnFastCopy: + pass + # Linux + elif _USE_CP_SENDFILE: + try: + _fastcopy_sendfile(fsrc, fdst) + return dst + except _GiveupOnFastCopy: + pass + # Windows, see: + # https://github.com/python/cpython/pull/7160#discussion_r195405230 + elif _WINDOWS and file_size > 0: + _copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE)) + return dst + + copyfileobj(fsrc, fdst) + + # Issue 43219, raise a less confusing exception + except IsADirectoryError as e: + if not os.path.exists(dst): + raise FileNotFoundError(f'Directory does not exist: {dst}') from e + else: + raise return dst @@ -296,11 +304,15 @@ def copymode(src, dst, *, follow_symlinks=True): sys.audit("shutil.copymode", src, dst) if not follow_symlinks and _islink(src) and os.path.islink(dst): - if hasattr(os, 'lchmod'): + if os.name == 'nt': + stat_func, chmod_func = os.lstat, os.chmod + elif hasattr(os, 'lchmod'): stat_func, chmod_func = os.lstat, os.lchmod else: return else: + if os.name == 'nt' and os.path.islink(dst): + dst = os.path.realpath(dst, strict=True) stat_func, chmod_func = _stat, os.chmod st = stat_func(src) @@ -328,7 +340,7 @@ def _copyxattr(src, dst, *, follow_symlinks=True): os.setxattr(dst, name, value, follow_symlinks=follow_symlinks) except OSError as e: if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA, - errno.EINVAL): + errno.EINVAL, errno.EACCES): raise else: def _copyxattr(*args, **kwargs): @@ -376,8 +388,16 @@ def lookup(name): # We must copy extended attributes before the file is (potentially) # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. _copyxattr(src, dst, follow_symlinks=follow) + _chmod = lookup("chmod") + if os.name == 'nt': + if follow: + if os.path.islink(dst): + dst = os.path.realpath(dst, strict=True) + else: + def _chmod(*args, **kwargs): + os.chmod(*args) try: - lookup("chmod")(dst, mode, follow_symlinks=follow) + _chmod(dst, mode, follow_symlinks=follow) except NotImplementedError: # if we got a NotImplementedError, it's because # * follow_symlinks=False, @@ -431,6 +451,29 @@ def copy2(src, dst, *, follow_symlinks=True): """ if os.path.isdir(dst): dst = os.path.join(dst, os.path.basename(src)) + + if hasattr(_winapi, "CopyFile2"): + src_ = os.fsdecode(src) + dst_ = os.fsdecode(dst) + flags = _winapi.COPY_FILE_ALLOW_DECRYPTED_DESTINATION # for compat + if not follow_symlinks: + flags |= _winapi.COPY_FILE_COPY_SYMLINK + try: + _winapi.CopyFile2(src_, dst_, flags) + return dst + except OSError as exc: + if (exc.winerror == _winapi.ERROR_PRIVILEGE_NOT_HELD + and not follow_symlinks): + # Likely encountered a symlink we aren't allowed to create. + # Fall back on the old code + pass + elif exc.winerror == _winapi.ERROR_ACCESS_DENIED: + # Possibly encountered a hidden or readonly file we can't + # overwrite. Fall back on old code + pass + else: + raise + copyfile(src, dst, follow_symlinks=follow_symlinks) copystat(src, dst, follow_symlinks=follow_symlinks) return dst @@ -452,7 +495,7 @@ def _copytree(entries, src, dst, symlinks, ignore, copy_function, if ignore is not None: ignored_names = ignore(os.fspath(src), [x.name for x in entries]) else: - ignored_names = set() + ignored_names = () os.makedirs(dst, exist_ok=dirs_exist_ok) errors = [] @@ -487,12 +530,13 @@ def _copytree(entries, src, dst, symlinks, ignore, copy_function, # otherwise let the copy occur. copy2 will raise an error if srcentry.is_dir(): copytree(srcobj, dstname, symlinks, ignore, - copy_function, dirs_exist_ok=dirs_exist_ok) + copy_function, ignore_dangling_symlinks, + dirs_exist_ok) else: copy_function(srcobj, dstname) elif srcentry.is_dir(): copytree(srcobj, dstname, symlinks, ignore, copy_function, - dirs_exist_ok=dirs_exist_ok) + ignore_dangling_symlinks, dirs_exist_ok) else: # Will raise a SpecialFileError for unsupported file types copy_function(srcobj, dstname) @@ -516,9 +560,6 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, ignore_dangling_symlinks=False, dirs_exist_ok=False): """Recursively copy a directory tree and return the destination directory. - dirs_exist_ok dictates whether to raise an exception in case dst or any - missing parent directory already exists. - If exception(s) occur, an Error is raised with a list of reasons. If the optional symlinks flag is true, symbolic links in the @@ -549,6 +590,11 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, destination path as arguments. By default, copy2() is used, but any function that supports the same signature (like copy()) can be used. + If dirs_exist_ok is false (the default) and `dst` already exists, a + `FileExistsError` is raised. If `dirs_exist_ok` is true, the copying + operation will continue if it encounters existing directories, and files + within the `dst` tree will be overwritten by corresponding files from the + `src` tree. """ sys.audit("shutil.copytree", src, dst) with os.scandir(src) as itr: @@ -559,18 +605,6 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, dirs_exist_ok=dirs_exist_ok) if hasattr(os.stat_result, 'st_file_attributes'): - # Special handling for directory junctions to make them behave like - # symlinks for shutil.rmtree, since in general they do not appear as - # regular links. - def _rmtree_isdir(entry): - try: - st = entry.stat(follow_symlinks=False) - return (stat.S_ISDIR(st.st_mode) and not - (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT - and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) - except OSError: - return False - def _rmtree_islink(path): try: st = os.lstat(path) @@ -580,54 +614,53 @@ def _rmtree_islink(path): except OSError: return False else: - def _rmtree_isdir(entry): - try: - return entry.is_dir(follow_symlinks=False) - except OSError: - return False - def _rmtree_islink(path): return os.path.islink(path) # version vulnerable to race conditions -def _rmtree_unsafe(path, onerror): +def _rmtree_unsafe(path, onexc): try: with os.scandir(path) as scandir_it: entries = list(scandir_it) - except OSError: - onerror(os.scandir, path, sys.exc_info()) + except OSError as err: + onexc(os.scandir, path, err) entries = [] for entry in entries: fullname = entry.path - if _rmtree_isdir(entry): + try: + is_dir = entry.is_dir(follow_symlinks=False) + except OSError: + is_dir = False + + if is_dir and not entry.is_junction(): try: if entry.is_symlink(): # This can only happen if someone replaces # a directory with a symlink after the call to # os.scandir or entry.is_dir above. raise OSError("Cannot call rmtree on a symbolic link") - except OSError: - onerror(os.path.islink, fullname, sys.exc_info()) + except OSError as err: + onexc(os.path.islink, fullname, err) continue - _rmtree_unsafe(fullname, onerror) + _rmtree_unsafe(fullname, onexc) else: try: os.unlink(fullname) - except OSError: - onerror(os.unlink, fullname, sys.exc_info()) + except OSError as err: + onexc(os.unlink, fullname, err) try: os.rmdir(path) - except OSError: - onerror(os.rmdir, path, sys.exc_info()) + except OSError as err: + onexc(os.rmdir, path, err) # Version using fd-based APIs to protect against races -def _rmtree_safe_fd(topfd, path, onerror): +def _rmtree_safe_fd(topfd, path, onexc): try: with os.scandir(topfd) as scandir_it: entries = list(scandir_it) except OSError as err: err.filename = path - onerror(os.scandir, path, sys.exc_info()) + onexc(os.scandir, path, err) return for entry in entries: fullname = os.path.join(path, entry.name) @@ -640,22 +673,30 @@ def _rmtree_safe_fd(topfd, path, onerror): try: orig_st = entry.stat(follow_symlinks=False) is_dir = stat.S_ISDIR(orig_st.st_mode) - except OSError: - onerror(os.lstat, fullname, sys.exc_info()) + except OSError as err: + onexc(os.lstat, fullname, err) continue if is_dir: try: - dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) - except OSError: - onerror(os.open, fullname, sys.exc_info()) + dirfd = os.open(entry.name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=topfd) + dirfd_closed = False + except OSError as err: + onexc(os.open, fullname, err) else: try: if os.path.samestat(orig_st, os.fstat(dirfd)): - _rmtree_safe_fd(dirfd, fullname, onerror) + _rmtree_safe_fd(dirfd, fullname, onexc) + try: + os.close(dirfd) + except OSError as err: + # close() should not be retried after an error. + dirfd_closed = True + onexc(os.close, fullname, err) + dirfd_closed = True try: os.rmdir(entry.name, dir_fd=topfd) - except OSError: - onerror(os.rmdir, fullname, sys.exc_info()) + except OSError as err: + onexc(os.rmdir, fullname, err) else: try: # This can only happen if someone replaces @@ -663,39 +704,67 @@ def _rmtree_safe_fd(topfd, path, onerror): # os.scandir or stat.S_ISDIR above. raise OSError("Cannot call rmtree on a symbolic " "link") - except OSError: - onerror(os.path.islink, fullname, sys.exc_info()) + except OSError as err: + onexc(os.path.islink, fullname, err) finally: - os.close(dirfd) + if not dirfd_closed: + try: + os.close(dirfd) + except OSError as err: + onexc(os.close, fullname, err) else: try: os.unlink(entry.name, dir_fd=topfd) - except OSError: - onerror(os.unlink, fullname, sys.exc_info()) + except OSError as err: + onexc(os.unlink, fullname, err) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and os.scandir in os.supports_fd and os.stat in os.supports_follow_symlinks) -def rmtree(path, ignore_errors=False, onerror=None): +def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): """Recursively delete a directory tree. - If ignore_errors is set, errors are ignored; otherwise, if onerror - is set, it is called to handle the error with arguments (func, + If dir_fd is not None, it should be a file descriptor open to a directory; + path will then be relative to that directory. + dir_fd may not be implemented on your platform. + If it is unavailable, using it will raise a NotImplementedError. + + If ignore_errors is set, errors are ignored; otherwise, if onexc or + onerror is set, it is called to handle the error with arguments (func, path, exc_info) where func is platform and implementation dependent; path is the argument to that function that caused it to fail; and - exc_info is a tuple returned by sys.exc_info(). If ignore_errors - is false and onerror is None, an exception is raised. + the value of exc_info describes the exception. For onexc it is the + exception instance, and for onerror it is a tuple as returned by + sys.exc_info(). If ignore_errors is false and both onexc and + onerror are None, the exception is reraised. + onerror is deprecated and only remains for backwards compatibility. + If both onerror and onexc are set, onerror is ignored and onexc is used. """ - sys.audit("shutil.rmtree", path) + + sys.audit("shutil.rmtree", path, dir_fd) if ignore_errors: - def onerror(*args): + def onexc(*args): pass - elif onerror is None: - def onerror(*args): + elif onerror is None and onexc is None: + def onexc(*args): raise + elif onexc is None: + if onerror is None: + def onexc(*args): + raise + else: + # delegate to onerror + def onexc(*args): + func, path, exc = args + if exc is None: + exc_info = None, None, None + else: + exc_info = type(exc), exc, exc.__traceback__ + return onerror(func, path, exc_info) + if _use_fd_functions: # While the unsafe rmtree works fine on bytes, the fd based does not. if isinstance(path, bytes): @@ -703,48 +772,74 @@ def onerror(*args): # Note: To guard against symlink races, we use the standard # lstat()/open()/fstat() trick. try: - orig_st = os.lstat(path) - except Exception: - onerror(os.lstat, path, sys.exc_info()) + orig_st = os.lstat(path, dir_fd=dir_fd) + except Exception as err: + onexc(os.lstat, path, err) return try: - fd = os.open(path, os.O_RDONLY) - except Exception: - onerror(os.lstat, path, sys.exc_info()) + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dir_fd) + fd_closed = False + except Exception as err: + onexc(os.open, path, err) return try: if os.path.samestat(orig_st, os.fstat(fd)): - _rmtree_safe_fd(fd, path, onerror) + _rmtree_safe_fd(fd, path, onexc) + try: + os.close(fd) + except OSError as err: + # close() should not be retried after an error. + fd_closed = True + onexc(os.close, path, err) + fd_closed = True try: - os.rmdir(path) - except OSError: - onerror(os.rmdir, path, sys.exc_info()) + os.rmdir(path, dir_fd=dir_fd) + except OSError as err: + onexc(os.rmdir, path, err) else: try: # symlinks to directories are forbidden, see bug #1669 raise OSError("Cannot call rmtree on a symbolic link") - except OSError: - onerror(os.path.islink, path, sys.exc_info()) + except OSError as err: + onexc(os.path.islink, path, err) finally: - os.close(fd) + if not fd_closed: + try: + os.close(fd) + except OSError as err: + onexc(os.close, path, err) else: + if dir_fd is not None: + raise NotImplementedError("dir_fd unavailable on this platform") try: if _rmtree_islink(path): # symlinks to directories are forbidden, see bug #1669 raise OSError("Cannot call rmtree on a symbolic link") - except OSError: - onerror(os.path.islink, path, sys.exc_info()) - # can't continue even if onerror hook returns + except OSError as err: + onexc(os.path.islink, path, err) + # can't continue even if onexc hook returns return - return _rmtree_unsafe(path, onerror) + return _rmtree_unsafe(path, onexc) # Allow introspection of whether or not the hardening against symlink # attacks is supported on the current platform rmtree.avoids_symlink_attacks = _use_fd_functions def _basename(path): - # A basename() variant which first strips the trailing slash, if present. - # Thus we always get the last component of the path, even for directories. + """A basename() variant which first strips the trailing slash, if present. + Thus we always get the last component of the path, even for directories. + + path: Union[PathLike, str] + + e.g. + >>> os.path.basename('/bar/foo') + 'foo' + >>> os.path.basename('/bar/foo/') + '' + >>> _basename('/bar/foo/') + 'foo' + """ + path = os.fspath(path) sep = os.path.sep + (os.path.altsep or '') return os.path.basename(path.rstrip(sep)) @@ -753,12 +848,12 @@ def move(src, dst, copy_function=copy2): similar to the Unix "mv" command. Return the file or directory's destination. - If the destination is a directory or a symlink to a directory, the source - is moved inside the directory. The destination path must not already - exist. + If dst is an existing directory or a symlink to a directory, then src is + moved inside that directory. The destination path in that directory must + not already exist. - If the destination already exists but is not a directory, it may be - overwritten depending on os.rename() semantics. + If dst already exists but is not a directory, it may be overwritten + depending on os.rename() semantics. If the destination is on our current filesystem, then rename() is used. Otherwise, src is copied to the destination and then removed. Symlinks are @@ -777,13 +872,16 @@ def move(src, dst, copy_function=copy2): sys.audit("shutil.move", src, dst) real_dst = dst if os.path.isdir(dst): - if _samefile(src, dst): + if _samefile(src, dst) and not os.path.islink(src): # We might be on a case insensitive filesystem, # perform the rename anyway. os.rename(src, dst) return + # Using _basename instead of os.path.basename is important, as we must + # ignore any trailing slash to avoid the basename returning '' real_dst = os.path.join(dst, _basename(src)) + if os.path.exists(real_dst): raise Error("Destination path '%s' already exists" % real_dst) try: @@ -797,6 +895,12 @@ def move(src, dst, copy_function=copy2): if _destinsrc(src, dst): raise Error("Cannot move a directory '%s' into itself" " '%s'." % (src, dst)) + if (_is_immutable(src) + or (not os.access(src, os.W_OK) and os.listdir(src) + and sys.platform == 'darwin')): + raise PermissionError("Cannot move the non-empty directory " + "'%s': Lacking write permission to '%s'." + % (src, src)) copytree(src, real_dst, copy_function=copy_function, symlinks=True) rmtree(src) @@ -814,10 +918,21 @@ def _destinsrc(src, dst): dst += os.path.sep return dst.startswith(src) +def _is_immutable(src): + st = _stat(src) + immutable_states = [stat.UF_IMMUTABLE, stat.SF_IMMUTABLE] + return hasattr(st, 'st_flags') and st.st_flags in immutable_states + def _get_gid(name): """Returns a gid, given a group name.""" - if getgrnam is None or name is None: + if name is None: + return None + + try: + from grp import getgrnam + except ImportError: return None + try: result = getgrnam(name) except KeyError: @@ -828,8 +943,14 @@ def _get_gid(name): def _get_uid(name): """Returns an uid, given a user name.""" - if getpwnam is None or name is None: + if name is None: return None + + try: + from pwd import getpwnam + except ImportError: + return None + try: result = getpwnam(name) except KeyError: @@ -839,7 +960,7 @@ def _get_uid(name): return None def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, - owner=None, group=None, logger=None): + owner=None, group=None, logger=None, root_dir=None): """Create a (possibly compressed) tar file from all the files under 'base_dir'. @@ -896,14 +1017,20 @@ def _set_uid_gid(tarinfo): if not dry_run: tar = tarfile.open(archive_name, 'w|%s' % tar_compression) + arcname = base_dir + if root_dir is not None: + base_dir = os.path.join(root_dir, base_dir) try: - tar.add(base_dir, filter=_set_uid_gid) + tar.add(base_dir, arcname, filter=_set_uid_gid) finally: tar.close() + if root_dir is not None: + archive_name = os.path.abspath(archive_name) return archive_name -def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): +def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, + logger=None, owner=None, group=None, root_dir=None): """Create a zip file from all the files under 'base_dir'. The output zip file will be named 'base_name' + ".zip". Returns the @@ -927,28 +1054,48 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): if not dry_run: with zipfile.ZipFile(zip_filename, "w", compression=zipfile.ZIP_DEFLATED) as zf: - path = os.path.normpath(base_dir) - if path != os.curdir: - zf.write(path, path) + arcname = os.path.normpath(base_dir) + if root_dir is not None: + base_dir = os.path.join(root_dir, base_dir) + base_dir = os.path.normpath(base_dir) + if arcname != os.curdir: + zf.write(base_dir, arcname) if logger is not None: - logger.info("adding '%s'", path) + logger.info("adding '%s'", base_dir) for dirpath, dirnames, filenames in os.walk(base_dir): + arcdirpath = dirpath + if root_dir is not None: + arcdirpath = os.path.relpath(arcdirpath, root_dir) + arcdirpath = os.path.normpath(arcdirpath) for name in sorted(dirnames): - path = os.path.normpath(os.path.join(dirpath, name)) - zf.write(path, path) + path = os.path.join(dirpath, name) + arcname = os.path.join(arcdirpath, name) + zf.write(path, arcname) if logger is not None: logger.info("adding '%s'", path) for name in filenames: - path = os.path.normpath(os.path.join(dirpath, name)) + path = os.path.join(dirpath, name) + path = os.path.normpath(path) if os.path.isfile(path): - zf.write(path, path) + arcname = os.path.join(arcdirpath, name) + zf.write(path, arcname) if logger is not None: logger.info("adding '%s'", path) + if root_dir is not None: + zip_filename = os.path.abspath(zip_filename) return zip_filename +_make_tarball.supports_root_dir = True +_make_zipfile.supports_root_dir = True + +# Maps the name of the archive format to a tuple containing: +# * the archiving function +# * extra keyword arguments +# * description _ARCHIVE_FORMATS = { - 'tar': (_make_tarball, [('compress', None)], "uncompressed tar file"), + 'tar': (_make_tarball, [('compress', None)], + "uncompressed tar file"), } if _ZLIB_SUPPORTED: @@ -1017,36 +1164,44 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, uses the current owner and group. """ sys.audit("shutil.make_archive", base_name, format, root_dir, base_dir) - save_cwd = os.getcwd() - if root_dir is not None: - if logger is not None: - logger.debug("changing into '%s'", root_dir) - base_name = os.path.abspath(base_name) - if not dry_run: - os.chdir(root_dir) - - if base_dir is None: - base_dir = os.curdir - - kwargs = {'dry_run': dry_run, 'logger': logger} - try: format_info = _ARCHIVE_FORMATS[format] except KeyError: raise ValueError("unknown archive format '%s'" % format) from None + kwargs = {'dry_run': dry_run, 'logger': logger, + 'owner': owner, 'group': group} + func = format_info[0] for arg, val in format_info[1]: kwargs[arg] = val - if format != 'zip': - kwargs['owner'] = owner - kwargs['group'] = group + if base_dir is None: + base_dir = os.curdir + + supports_root_dir = getattr(func, 'supports_root_dir', False) + save_cwd = None + if root_dir is not None: + stmd = os.stat(root_dir).st_mode + if not stat.S_ISDIR(stmd): + raise NotADirectoryError(errno.ENOTDIR, 'Not a directory', root_dir) + + if supports_root_dir: + # Support path-like base_name here for backwards-compatibility. + base_name = os.fspath(base_name) + kwargs['root_dir'] = root_dir + else: + save_cwd = os.getcwd() + if logger is not None: + logger.debug("changing into '%s'", root_dir) + base_name = os.path.abspath(base_name) + if not dry_run: + os.chdir(root_dir) try: filename = func(base_name, base_dir, **kwargs) finally: - if root_dir is not None: + if save_cwd is not None: if logger is not None: logger.debug("changing back to '%s'", save_cwd) os.chdir(save_cwd) @@ -1132,24 +1287,20 @@ def _unpack_zipfile(filename, extract_dir): if name.startswith('/') or '..' in name: continue - target = os.path.join(extract_dir, *name.split('/')) - if not target: + targetpath = os.path.join(extract_dir, *name.split('/')) + if not targetpath: continue - _ensure_directory(target) + _ensure_directory(targetpath) if not name.endswith('/'): # file - data = zip.read(info.filename) - f = open(target, 'wb') - try: - f.write(data) - finally: - f.close() - del data + with zip.open(name, 'r') as source, \ + open(targetpath, 'wb') as target: + copyfileobj(source, target) finally: zip.close() -def _unpack_tarfile(filename, extract_dir): +def _unpack_tarfile(filename, extract_dir, *, filter=None): """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` """ import tarfile # late import for breaking circular dependency @@ -1159,10 +1310,15 @@ def _unpack_tarfile(filename, extract_dir): raise ReadError( "%s is not a compressed or uncompressed tar file" % filename) try: - tarobj.extractall(extract_dir) + tarobj.extractall(extract_dir, filter=filter) finally: tarobj.close() +# Maps the name of the unpack format to a tuple containing: +# * extensions +# * the unpacking function +# * extra keyword arguments +# * description _UNPACK_FORMATS = { 'tar': (['.tar'], _unpack_tarfile, [], "uncompressed tar file"), 'zip': (['.zip'], _unpack_zipfile, [], "ZIP file"), @@ -1187,7 +1343,7 @@ def _find_unpack_format(filename): return name return None -def unpack_archive(filename, extract_dir=None, format=None): +def unpack_archive(filename, extract_dir=None, format=None, *, filter=None): """Unpack an archive. `filename` is the name of the archive. @@ -1201,6 +1357,9 @@ def unpack_archive(filename, extract_dir=None, format=None): was registered for that extension. In case none is found, a ValueError is raised. + + If `filter` is given, it is passed to the underlying + extraction function. """ sys.audit("shutil.unpack_archive", filename, extract_dir, format) @@ -1210,6 +1369,10 @@ def unpack_archive(filename, extract_dir=None, format=None): extract_dir = os.fspath(extract_dir) filename = os.fspath(filename) + if filter is None: + filter_kwargs = {} + else: + filter_kwargs = {'filter': filter} if format is not None: try: format_info = _UNPACK_FORMATS[format] @@ -1217,7 +1380,7 @@ def unpack_archive(filename, extract_dir=None, format=None): raise ValueError("Unknown unpack format '{0}'".format(format)) from None func = format_info[1] - func(filename, extract_dir, **dict(format_info[2])) + func(filename, extract_dir, **dict(format_info[2]), **filter_kwargs) else: # we need to look at the registered unpackers supported extensions format = _find_unpack_format(filename) @@ -1225,7 +1388,7 @@ def unpack_archive(filename, extract_dir=None, format=None): raise ReadError("Unknown archive format '{0}'".format(filename)) func = _UNPACK_FORMATS[format][1] - kwargs = dict(_UNPACK_FORMATS[format][2]) + kwargs = dict(_UNPACK_FORMATS[format][2]) | filter_kwargs func(filename, extract_dir, **kwargs) @@ -1336,9 +1499,9 @@ def get_terminal_size(fallback=(80, 24)): # os.get_terminal_size() is unsupported size = os.terminal_size(fallback) if columns <= 0: - columns = size.columns + columns = size.columns or fallback[0] if lines <= 0: - lines = size.lines + lines = size.lines or fallback[1] return os.terminal_size((columns, lines)) @@ -1351,6 +1514,16 @@ def _access_check(fn, mode): and not os.path.isdir(fn)) +def _win_path_needs_curdir(cmd, mode): + """ + On Windows, we can use NeedCurrentDirectoryForExePath to figure out + if we should add the cwd to PATH when searching for executables if + the mode is executable. + """ + return (not (mode & os.X_OK)) or _winapi.NeedCurrentDirectoryForExePath( + os.fsdecode(cmd)) + + def which(cmd, mode=os.F_OK | os.X_OK, path=None): """Given a command, mode, and a PATH string, return the path which conforms to the given mode on the PATH, or None if there is no such @@ -1361,58 +1534,62 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): path. """ - # If we're given a path with a directory part, look it up directly rather - # than referring to PATH directories. This includes checking relative to the - # current directory, e.g. ./script - if os.path.dirname(cmd): - if _access_check(cmd, mode): - return cmd - return None - use_bytes = isinstance(cmd, bytes) - if path is None: - path = os.environ.get("PATH", None) - if path is None: - try: - path = os.confstr("CS_PATH") - except (AttributeError, ValueError): - # os.confstr() or CS_PATH is not available - path = os.defpath - # bpo-35755: Don't use os.defpath if the PATH environment variable is - # set to an empty string - - # PATH='' doesn't match, whereas PATH=':' looks in the current directory - if not path: - return None - - if use_bytes: - path = os.fsencode(path) - path = path.split(os.fsencode(os.pathsep)) + # If we're given a path with a directory part, look it up directly rather + # than referring to PATH directories. This includes checking relative to + # the current directory, e.g. ./script + dirname, cmd = os.path.split(cmd) + if dirname: + path = [dirname] else: - path = os.fsdecode(path) - path = path.split(os.pathsep) + if path is None: + path = os.environ.get("PATH", None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable + # is set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current + # directory + if not path: + return None - if sys.platform == "win32": - # The current directory takes precedence on Windows. - curdir = os.curdir if use_bytes: - curdir = os.fsencode(curdir) - if curdir not in path: + path = os.fsencode(path) + path = path.split(os.fsencode(os.pathsep)) + else: + path = os.fsdecode(path) + path = path.split(os.pathsep) + + if sys.platform == "win32" and _win_path_needs_curdir(cmd, mode): + curdir = os.curdir + if use_bytes: + curdir = os.fsencode(curdir) path.insert(0, curdir) + if sys.platform == "win32": # PATHEXT is necessary to check on Windows. - pathext = os.environ.get("PATHEXT", "").split(os.pathsep) + pathext_source = os.getenv("PATHEXT") or _WIN_DEFAULT_PATHEXT + pathext = [ext for ext in pathext_source.split(os.pathsep) if ext] + if use_bytes: pathext = [os.fsencode(ext) for ext in pathext] - # See if the given file matches any of the expected path extensions. - # This will allow us to short circuit when given "python.exe". - # If it does match, only test that one, otherwise we have to try - # others. - if any(cmd.lower().endswith(ext.lower()) for ext in pathext): - files = [cmd] - else: - files = [cmd + ext for ext in pathext] + + files = ([cmd] + [cmd + ext for ext in pathext]) + + # gh-109590. If we are looking for an executable, we need to look + # for a PATHEXT match. The first cmd is the direct match + # (e.g. python.exe instead of python) + # Check that direct match first if and only if the extension is in PATHEXT + # Otherwise check it last + suffix = os.path.splitext(files[0])[1].upper() + if mode & os.X_OK and not any(suffix == ext.upper() for ext in pathext): + files.append(files.pop(0)) else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index fba98972dd..1342355069 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -23,6 +23,7 @@ unregister_unpack_format, get_unpack_formats, SameFileError, _GiveupOnFastCopy) import tarfile +import warnings import zipfile try: import posix @@ -32,9 +33,13 @@ from test import support from test.support import os_helper from test.support.os_helper import TESTFN, FakePath +from test.support import warnings_helper TESTFN2 = TESTFN + "2" +TESTFN_SRC = TESTFN + "_SRC" +TESTFN_DST = TESTFN + "_DST" MACOS = sys.platform.startswith("darwin") +SOLARIS = sys.platform.startswith("sunos") AIX = sys.platform[:3] == 'aix' try: import grp @@ -48,6 +53,9 @@ except ImportError: _winapi = None +no_chdir = unittest.mock.patch('os.chdir', + side_effect=AssertionError("shouldn't call os.chdir()")) + def _fake_rename(*args, **kwargs): # Pretend the destination path is on a different filesystem. raise OSError(getattr(errno, 'EXDEV', 18), "Invalid cross-device link") @@ -72,7 +80,9 @@ def write_file(path, content, binary=False): """ if isinstance(path, tuple): path = os.path.join(*path) - with open(path, 'wb' if binary else 'w') as fp: + mode = 'wb' if binary else 'w' + encoding = None if binary else "utf-8" + with open(path, mode, encoding=encoding) as fp: fp.write(content) def write_test_file(path, size): @@ -102,7 +112,9 @@ def read_file(path, binary=False): """ if isinstance(path, tuple): path = os.path.join(*path) - with open(path, 'rb' if binary else 'r') as fp: + mode = 'rb' if binary else 'r' + encoding = None if binary else "utf-8" + with open(path, mode, encoding=encoding) as fp: return fp.read() def rlistdir(path): @@ -124,12 +136,12 @@ def supports_file2file_sendfile(): srcname = None dstname = None try: - with tempfile.NamedTemporaryFile("wb", delete=False) as f: + with tempfile.NamedTemporaryFile("wb", dir=os.getcwd(), delete=False) as f: srcname = f.name f.write(b"0123456789") with open(srcname, "rb") as src: - with tempfile.NamedTemporaryFile("wb", delete=False) as dst: + with tempfile.NamedTemporaryFile("wb", dir=os.getcwd(), delete=False) as dst: dstname = dst.name infd = src.fileno() outfd = dst.fileno() @@ -160,31 +172,21 @@ def _maxdataOK(): else: return True -class TestShutil(unittest.TestCase): - - def setUp(self): - super(TestShutil, self).setUp() - self.tempdirs = [] - - def tearDown(self): - super(TestShutil, self).tearDown() - while self.tempdirs: - d = self.tempdirs.pop() - shutil.rmtree(d, os.name in ('nt', 'cygwin')) +class BaseTest: - def mkdtemp(self): + def mkdtemp(self, prefix=None): """Create a temporary directory that will be cleaned up. Returns the path of the directory. """ - basedir = None - if sys.platform == "win32": - basedir = os.path.realpath(os.getcwd()) - d = tempfile.mkdtemp(dir=basedir) - self.tempdirs.append(d) + d = tempfile.mkdtemp(prefix=prefix, dir=os.getcwd()) + self.addCleanup(os_helper.rmtree, d) return d + +class TestRmTree(BaseTest, unittest.TestCase): + def test_rmtree_works_on_bytes(self): tmp = self.mkdtemp() victim = os.path.join(tmp, 'killme') @@ -195,7 +197,7 @@ def test_rmtree_works_on_bytes(self): shutil.rmtree(victim) @os_helper.skip_unless_symlink - def test_rmtree_fails_on_symlink(self): + def test_rmtree_fails_on_symlink_onerror(self): tmp = self.mkdtemp() dir_ = os.path.join(tmp, 'dir') os.mkdir(dir_) @@ -213,6 +215,25 @@ def onerror(*args): self.assertEqual(errors[0][1], link) self.assertIsInstance(errors[0][2][1], OSError) + @os_helper.skip_unless_symlink + def test_rmtree_fails_on_symlink_onexc(self): + tmp = self.mkdtemp() + dir_ = os.path.join(tmp, 'dir') + os.mkdir(dir_) + link = os.path.join(tmp, 'link') + os.symlink(dir_, link) + self.assertRaises(OSError, shutil.rmtree, link) + self.assertTrue(os.path.exists(dir_)) + self.assertTrue(os.path.lexists(link)) + errors = [] + def onexc(*args): + errors.append(args) + shutil.rmtree(link, onexc=onexc) + self.assertEqual(len(errors), 1) + self.assertIs(errors[0][0], os.path.islink) + self.assertEqual(errors[0][1], link) + self.assertIsInstance(errors[0][2], OSError) + @os_helper.skip_unless_symlink def test_rmtree_works_on_symlinks(self): tmp = self.mkdtemp() @@ -236,12 +257,13 @@ def test_rmtree_works_on_symlinks(self): self.assertTrue(os.path.exists(file1)) @unittest.skipUnless(_winapi, 'only relevant on Windows') - def test_rmtree_fails_on_junctions(self): + def test_rmtree_fails_on_junctions_onerror(self): tmp = self.mkdtemp() dir_ = os.path.join(tmp, 'dir') os.mkdir(dir_) link = os.path.join(tmp, 'link') _winapi.CreateJunction(dir_, link) + self.addCleanup(os_helper.unlink, link) self.assertRaises(OSError, shutil.rmtree, link) self.assertTrue(os.path.exists(dir_)) self.assertTrue(os.path.lexists(link)) @@ -254,6 +276,26 @@ def onerror(*args): self.assertEqual(errors[0][1], link) self.assertIsInstance(errors[0][2][1], OSError) + @unittest.skipUnless(_winapi, 'only relevant on Windows') + def test_rmtree_fails_on_junctions_onexc(self): + tmp = self.mkdtemp() + dir_ = os.path.join(tmp, 'dir') + os.mkdir(dir_) + link = os.path.join(tmp, 'link') + _winapi.CreateJunction(dir_, link) + self.addCleanup(os_helper.unlink, link) + self.assertRaises(OSError, shutil.rmtree, link) + self.assertTrue(os.path.exists(dir_)) + self.assertTrue(os.path.lexists(link)) + errors = [] + def onexc(*args): + errors.append(args) + shutil.rmtree(link, onexc=onexc) + self.assertEqual(len(errors), 1) + self.assertIs(errors[0][0], os.path.islink) + self.assertEqual(errors[0][1], link) + self.assertIsInstance(errors[0][2], OSError) + @unittest.skipUnless(_winapi, 'only relevant on Windows') def test_rmtree_works_on_junctions(self): tmp = self.mkdtemp() @@ -278,9 +320,9 @@ def test_rmtree_works_on_junctions(self): # TODO: RUSTPYTHON @unittest.expectedFailure - def test_rmtree_errors(self): + def test_rmtree_errors_onerror(self): # filename is guaranteed not to exist - filename = tempfile.mktemp() + filename = tempfile.mktemp(dir=self.mkdtemp()) self.assertRaises(FileNotFoundError, shutil.rmtree, filename) # test that ignore_errors option is honored shutil.rmtree(filename, ignore_errors=True) @@ -291,10 +333,7 @@ def test_rmtree_errors(self): filename = os.path.join(tmpdir, "tstfile") with self.assertRaises(NotADirectoryError) as cm: shutil.rmtree(filename) - # The reason for this rather odd construct is that Windows sprinkles - # a \*.* at the end of file names. But only sometimes on some buildbots - possible_args = [filename, os.path.join(filename, '*.*')] - self.assertIn(cm.exception.filename, possible_args) + self.assertEqual(cm.exception.filename, filename) self.assertTrue(os.path.exists(filename)) # test that ignore_errors option is honored shutil.rmtree(filename, ignore_errors=True) @@ -307,17 +346,50 @@ def onerror(*args): self.assertIs(errors[0][0], os.scandir) self.assertEqual(errors[0][1], filename) self.assertIsInstance(errors[0][2][1], NotADirectoryError) - self.assertIn(errors[0][2][1].filename, possible_args) + self.assertEqual(errors[0][2][1].filename, filename) self.assertIs(errors[1][0], os.rmdir) self.assertEqual(errors[1][1], filename) self.assertIsInstance(errors[1][2][1], NotADirectoryError) - self.assertIn(errors[1][2][1].filename, possible_args) + self.assertEqual(errors[1][2][1].filename, filename) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_rmtree_errors_onexc(self): + # filename is guaranteed not to exist + filename = tempfile.mktemp(dir=self.mkdtemp()) + self.assertRaises(FileNotFoundError, shutil.rmtree, filename) + # test that ignore_errors option is honored + shutil.rmtree(filename, ignore_errors=True) + # existing file + tmpdir = self.mkdtemp() + write_file((tmpdir, "tstfile"), "") + filename = os.path.join(tmpdir, "tstfile") + with self.assertRaises(NotADirectoryError) as cm: + shutil.rmtree(filename) + self.assertEqual(cm.exception.filename, filename) + self.assertTrue(os.path.exists(filename)) + # test that ignore_errors option is honored + shutil.rmtree(filename, ignore_errors=True) + self.assertTrue(os.path.exists(filename)) + errors = [] + def onexc(*args): + errors.append(args) + shutil.rmtree(filename, onexc=onexc) + self.assertEqual(len(errors), 2) + self.assertIs(errors[0][0], os.scandir) + self.assertEqual(errors[0][1], filename) + self.assertIsInstance(errors[0][2], NotADirectoryError) + self.assertEqual(errors[0][2].filename, filename) + self.assertIs(errors[1][0], os.rmdir) + self.assertEqual(errors[1][1], filename) + self.assertIsInstance(errors[1][2], NotADirectoryError) + self.assertEqual(errors[1][2].filename, filename) @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") - @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, - "This test can't be run reliably as root (issue #1076467).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod def test_on_error(self): self.errorState = 0 os.mkdir(TESTFN) @@ -372,6 +444,104 @@ def check_args_to_onerror(self, func, arg, exc): self.assertTrue(issubclass(exc[0], OSError)) self.errorState = 3 + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_on_exc(self): + self.errorState = 0 + os.mkdir(TESTFN) + self.addCleanup(shutil.rmtree, TESTFN) + + self.child_file_path = os.path.join(TESTFN, 'a') + self.child_dir_path = os.path.join(TESTFN, 'b') + os_helper.create_empty_file(self.child_file_path) + os.mkdir(self.child_dir_path) + old_dir_mode = os.stat(TESTFN).st_mode + old_child_file_mode = os.stat(self.child_file_path).st_mode + old_child_dir_mode = os.stat(self.child_dir_path).st_mode + # Make unwritable. + new_mode = stat.S_IREAD|stat.S_IEXEC + os.chmod(self.child_file_path, new_mode) + os.chmod(self.child_dir_path, new_mode) + os.chmod(TESTFN, new_mode) + + self.addCleanup(os.chmod, TESTFN, old_dir_mode) + self.addCleanup(os.chmod, self.child_file_path, old_child_file_mode) + self.addCleanup(os.chmod, self.child_dir_path, old_child_dir_mode) + + shutil.rmtree(TESTFN, onexc=self.check_args_to_onexc) + # Test whether onexc has actually been called. + self.assertEqual(self.errorState, 3, + "Expected call to onexc function did not happen.") + + def check_args_to_onexc(self, func, arg, exc): + # test_rmtree_errors deliberately runs rmtree + # on a directory that is chmod 500, which will fail. + # This function is run when shutil.rmtree fails. + # 99.9% of the time it initially fails to remove + # a file in the directory, so the first time through + # func is os.remove. + # However, some Linux machines running ZFS on + # FUSE experienced a failure earlier in the process + # at os.listdir. The first failure may legally + # be either. + if self.errorState < 2: + if func is os.unlink: + self.assertEqual(arg, self.child_file_path) + elif func is os.rmdir: + self.assertEqual(arg, self.child_dir_path) + else: + self.assertIs(func, os.listdir) + self.assertIn(arg, [TESTFN, self.child_dir_path]) + self.assertTrue(isinstance(exc, OSError)) + self.errorState += 1 + else: + self.assertEqual(func, os.rmdir) + self.assertEqual(arg, TESTFN) + self.assertTrue(isinstance(exc, OSError)) + self.errorState = 3 + + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_both_onerror_and_onexc(self): + onerror_called = False + onexc_called = False + + def onerror(*args): + nonlocal onerror_called + onerror_called = True + + def onexc(*args): + nonlocal onexc_called + onexc_called = True + + os.mkdir(TESTFN) + self.addCleanup(shutil.rmtree, TESTFN) + + self.child_file_path = os.path.join(TESTFN, 'a') + self.child_dir_path = os.path.join(TESTFN, 'b') + os_helper.create_empty_file(self.child_file_path) + os.mkdir(self.child_dir_path) + old_dir_mode = os.stat(TESTFN).st_mode + old_child_file_mode = os.stat(self.child_file_path).st_mode + old_child_dir_mode = os.stat(self.child_dir_path).st_mode + # Make unwritable. + new_mode = stat.S_IREAD|stat.S_IEXEC + os.chmod(self.child_file_path, new_mode) + os.chmod(self.child_dir_path, new_mode) + os.chmod(TESTFN, new_mode) + + self.addCleanup(os.chmod, TESTFN, old_dir_mode) + self.addCleanup(os.chmod, self.child_file_path, old_child_file_mode) + self.addCleanup(os.chmod, self.child_dir_path, old_child_dir_mode) + + shutil.rmtree(TESTFN, onerror=onerror, onexc=onexc) + self.assertTrue(onexc_called) + self.assertFalse(onerror_called) + def test_rmtree_does_not_choke_on_failing_lstat(self): try: orig_lstat = os.lstat @@ -388,115 +558,613 @@ def raiser(fn, *args, **kwargs): finally: os.lstat = orig_lstat - @os_helper.skip_unless_symlink - def test_copymode_follow_symlinks(self): + def test_rmtree_uses_safe_fd_version_if_available(self): + _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= + os.supports_dir_fd and + os.listdir in os.supports_fd and + os.stat in os.supports_follow_symlinks) + if _use_fd_functions: + self.assertTrue(shutil._use_fd_functions) + self.assertTrue(shutil.rmtree.avoids_symlink_attacks) + tmp_dir = self.mkdtemp() + d = os.path.join(tmp_dir, 'a') + os.mkdir(d) + try: + real_rmtree = shutil._rmtree_safe_fd + class Called(Exception): pass + def _raiser(*args, **kwargs): + raise Called + shutil._rmtree_safe_fd = _raiser + self.assertRaises(Called, shutil.rmtree, d) + finally: + shutil._rmtree_safe_fd = real_rmtree + else: + self.assertFalse(shutil._use_fd_functions) + self.assertFalse(shutil.rmtree.avoids_symlink_attacks) + + @unittest.skipUnless(shutil._use_fd_functions, "requires safe rmtree") + def test_rmtree_fails_on_close(self): + # Test that the error handler is called for failed os.close() and that + # os.close() is only called once for a file descriptor. + tmp = self.mkdtemp() + dir1 = os.path.join(tmp, 'dir1') + os.mkdir(dir1) + dir2 = os.path.join(dir1, 'dir2') + os.mkdir(dir2) + def close(fd): + orig_close(fd) + nonlocal close_count + close_count += 1 + raise OSError + + close_count = 0 + with support.swap_attr(os, 'close', close) as orig_close: + with self.assertRaises(OSError): + shutil.rmtree(dir1) + self.assertTrue(os.path.isdir(dir2)) + self.assertEqual(close_count, 2) + + close_count = 0 + errors = [] + def onexc(*args): + errors.append(args) + with support.swap_attr(os, 'close', close) as orig_close: + shutil.rmtree(dir1, onexc=onexc) + self.assertEqual(len(errors), 2) + self.assertIs(errors[0][0], close) + self.assertEqual(errors[0][1], dir2) + self.assertIs(errors[1][0], close) + self.assertEqual(errors[1][1], dir1) + self.assertEqual(close_count, 2) + + @unittest.skipUnless(shutil._use_fd_functions, "dir_fd is not supported") + def test_rmtree_with_dir_fd(self): tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') - os.symlink(src, src_link) - os.symlink(dst, dst_link) - os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) - # file to file - os.chmod(dst, stat.S_IRWXO) - self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - shutil.copymode(src, dst) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # On Windows, os.chmod does not follow symlinks (issue #15411) - if os.name != 'nt': - # follow src link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow dst link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow both links - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - - @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod') - @os_helper.skip_unless_symlink - def test_copymode_symlink_to_symlink(self): + victim = 'killme' + fullname = os.path.join(tmp_dir, victim) + dir_fd = os.open(tmp_dir, os.O_RDONLY) + self.addCleanup(os.close, dir_fd) + os.mkdir(fullname) + os.mkdir(os.path.join(fullname, 'subdir')) + write_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') + self.assertTrue(os.path.exists(fullname)) + shutil.rmtree(victim, dir_fd=dir_fd) + self.assertFalse(os.path.exists(fullname)) + + @unittest.skipIf(shutil._use_fd_functions, "dir_fd is supported") + def test_rmtree_with_dir_fd_unsupported(self): tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') - os.symlink(src, src_link) - os.symlink(dst, dst_link) - os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) - os.chmod(dst, stat.S_IRWXU) - os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) - # link to link - os.lchmod(dst_link, stat.S_IRWXO) - shutil.copymode(src_link, dst_link, follow_symlinks=False) - self.assertEqual(os.lstat(src_link).st_mode, - os.lstat(dst_link).st_mode) - self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # src link - use chmod - os.lchmod(dst_link, stat.S_IRWXO) - shutil.copymode(src_link, dst, follow_symlinks=False) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # dst link - use chmod - os.lchmod(dst_link, stat.S_IRWXO) - shutil.copymode(src, dst_link, follow_symlinks=False) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + with self.assertRaises(NotImplementedError): + shutil.rmtree(tmp_dir, dir_fd=0) + self.assertTrue(os.path.exists(tmp_dir)) - @unittest.skipIf(hasattr(os, 'lchmod'), 'requires os.lchmod to be missing') - @os_helper.skip_unless_symlink - def test_copymode_symlink_to_symlink_wo_lchmod(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') - os.symlink(src, src_link) - os.symlink(dst, dst_link) - shutil.copymode(src_link, dst_link, follow_symlinks=False) # silent fail + def test_rmtree_dont_delete_file(self): + # When called on a file instead of a directory, don't delete it. + handle, path = tempfile.mkstemp(dir=self.mkdtemp()) + os.close(handle) + self.assertRaises(NotADirectoryError, shutil.rmtree, path) + os.remove(path) @os_helper.skip_unless_symlink - def test_copystat_symlinks(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - dst_link = os.path.join(tmp_dir, 'qux') - write_file(src, 'foo') - src_stat = os.stat(src) - os.utime(src, (src_stat.st_atime, - src_stat.st_mtime - 42.0)) # ensure different mtimes - write_file(dst, 'bar') - self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime) - os.symlink(src, src_link) - os.symlink(dst, dst_link) - if hasattr(os, 'lchmod'): - os.lchmod(src_link, stat.S_IRWXO) - if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): - os.lchflags(src_link, stat.UF_NODUMP) - src_link_stat = os.lstat(src_link) - # follow - if hasattr(os, 'lchmod'): - shutil.copystat(src_link, dst_link, follow_symlinks=True) - self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode) - # don't follow - shutil.copystat(src_link, dst_link, follow_symlinks=False) - dst_link_stat = os.lstat(dst_link) - if os.utime in os.supports_follow_symlinks: - for attr in 'st_atime', 'st_mtime': - # The modification times may be truncated in the new file. - self.assertLessEqual(getattr(src_link_stat, attr), - getattr(dst_link_stat, attr) + 1) + def test_rmtree_on_symlink(self): + # bug 1669. + os.mkdir(TESTFN) + try: + src = os.path.join(TESTFN, 'cheese') + dst = os.path.join(TESTFN, 'shop') + os.mkdir(src) + os.symlink(src, dst) + self.assertRaises(OSError, shutil.rmtree, dst) + shutil.rmtree(dst, ignore_errors=True) + finally: + shutil.rmtree(TESTFN, ignore_errors=True) + + @unittest.skipUnless(_winapi, 'only relevant on Windows') + def test_rmtree_on_junction(self): + os.mkdir(TESTFN) + try: + src = os.path.join(TESTFN, 'cheese') + dst = os.path.join(TESTFN, 'shop') + os.mkdir(src) + open(os.path.join(src, 'spam'), 'wb').close() + _winapi.CreateJunction(src, dst) + self.assertRaises(OSError, shutil.rmtree, dst) + shutil.rmtree(dst, ignore_errors=True) + finally: + shutil.rmtree(TESTFN, ignore_errors=True) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_rmtree_on_named_pipe(self): + os.mkfifo(TESTFN) + try: + with self.assertRaises(NotADirectoryError): + shutil.rmtree(TESTFN) + self.assertTrue(os.path.exists(TESTFN)) + finally: + os.unlink(TESTFN) + + os.mkdir(TESTFN) + os.mkfifo(os.path.join(TESTFN, 'mypipe')) + shutil.rmtree(TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + + +class TestCopyTree(BaseTest, unittest.TestCase): + + def test_copytree_simple(self): + src_dir = self.mkdtemp() + dst_dir = os.path.join(self.mkdtemp(), 'destination') + self.addCleanup(shutil.rmtree, src_dir) + self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) + write_file((src_dir, 'test.txt'), '123') + os.mkdir(os.path.join(src_dir, 'test_dir')) + write_file((src_dir, 'test_dir', 'test.txt'), '456') + + shutil.copytree(src_dir, dst_dir) + self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) + self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir'))) + self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir', + 'test.txt'))) + actual = read_file((dst_dir, 'test.txt')) + self.assertEqual(actual, '123') + actual = read_file((dst_dir, 'test_dir', 'test.txt')) + self.assertEqual(actual, '456') + + def test_copytree_dirs_exist_ok(self): + src_dir = self.mkdtemp() + dst_dir = self.mkdtemp() + self.addCleanup(shutil.rmtree, src_dir) + self.addCleanup(shutil.rmtree, dst_dir) + + write_file((src_dir, 'nonexisting.txt'), '123') + os.mkdir(os.path.join(src_dir, 'existing_dir')) + os.mkdir(os.path.join(dst_dir, 'existing_dir')) + write_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') + write_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') + + shutil.copytree(src_dir, dst_dir, dirs_exist_ok=True) + self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'nonexisting.txt'))) + self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'existing_dir'))) + self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'existing_dir', + 'existing.txt'))) + actual = read_file((dst_dir, 'nonexisting.txt')) + self.assertEqual(actual, '123') + actual = read_file((dst_dir, 'existing_dir', 'existing.txt')) + self.assertEqual(actual, 'has been replaced') + + with self.assertRaises(FileExistsError): + shutil.copytree(src_dir, dst_dir, dirs_exist_ok=False) + + @os_helper.skip_unless_symlink + def test_copytree_symlinks(self): + tmp_dir = self.mkdtemp() + src_dir = os.path.join(tmp_dir, 'src') + dst_dir = os.path.join(tmp_dir, 'dst') + sub_dir = os.path.join(src_dir, 'sub') + os.mkdir(src_dir) + os.mkdir(sub_dir) + write_file((src_dir, 'file.txt'), 'foo') + src_link = os.path.join(sub_dir, 'link') + dst_link = os.path.join(dst_dir, 'sub/link') + os.symlink(os.path.join(src_dir, 'file.txt'), + src_link) if hasattr(os, 'lchmod'): + os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(src_link, stat.UF_NODUMP) + src_stat = os.lstat(src_link) + shutil.copytree(src_dir, dst_dir, symlinks=True) + self.assertTrue(os.path.islink(os.path.join(dst_dir, 'sub', 'link'))) + actual = os.readlink(os.path.join(dst_dir, 'sub', 'link')) + # Bad practice to blindly strip the prefix as it may be required to + # correctly refer to the file, but we're only comparing paths here. + if os.name == 'nt' and actual.startswith('\\\\?\\'): + actual = actual[4:] + self.assertEqual(actual, os.path.join(src_dir, 'file.txt')) + dst_stat = os.lstat(dst_link) + if hasattr(os, 'lchmod'): + self.assertEqual(dst_stat.st_mode, src_stat.st_mode) + if hasattr(os, 'lchflags'): + self.assertEqual(dst_stat.st_flags, src_stat.st_flags) + + def test_copytree_with_exclude(self): + # creating data + join = os.path.join + exists = os.path.exists + src_dir = self.mkdtemp() + try: + dst_dir = join(self.mkdtemp(), 'destination') + write_file((src_dir, 'test.txt'), '123') + write_file((src_dir, 'test.tmp'), '123') + os.mkdir(join(src_dir, 'test_dir')) + write_file((src_dir, 'test_dir', 'test.txt'), '456') + os.mkdir(join(src_dir, 'test_dir2')) + write_file((src_dir, 'test_dir2', 'test.txt'), '456') + os.mkdir(join(src_dir, 'test_dir2', 'subdir')) + os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) + write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') + write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') + + # testing glob-like patterns + try: + patterns = shutil.ignore_patterns('*.tmp', 'test_dir2') + shutil.copytree(src_dir, dst_dir, ignore=patterns) + # checking the result: some elements should not be copied + self.assertTrue(exists(join(dst_dir, 'test.txt'))) + self.assertFalse(exists(join(dst_dir, 'test.tmp'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2'))) + finally: + shutil.rmtree(dst_dir) + try: + patterns = shutil.ignore_patterns('*.tmp', 'subdir*') + shutil.copytree(src_dir, dst_dir, ignore=patterns) + # checking the result: some elements should not be copied + self.assertFalse(exists(join(dst_dir, 'test.tmp'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir'))) + finally: + shutil.rmtree(dst_dir) + + # testing callable-style + try: + def _filter(src, names): + res = [] + for name in names: + path = os.path.join(src, name) + + if (os.path.isdir(path) and + path.split()[-1] == 'subdir'): + res.append(name) + elif os.path.splitext(path)[-1] in ('.py'): + res.append(name) + return res + + shutil.copytree(src_dir, dst_dir, ignore=_filter) + + # checking the result: some elements should not be copied + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2', + 'test.py'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir'))) + + finally: + shutil.rmtree(dst_dir) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(os.path.dirname(dst_dir)) + + def test_copytree_arg_types_of_ignore(self): + join = os.path.join + exists = os.path.exists + + tmp_dir = self.mkdtemp() + src_dir = join(tmp_dir, "source") + + os.mkdir(join(src_dir)) + os.mkdir(join(src_dir, 'test_dir')) + os.mkdir(os.path.join(src_dir, 'test_dir', 'subdir')) + write_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') + + invokations = [] + + def _ignore(src, names): + invokations.append(src) + self.assertIsInstance(src, str) + self.assertIsInstance(names, list) + self.assertEqual(len(names), len(set(names))) + for name in names: + self.assertIsInstance(name, str) + return [] + + dst_dir = join(self.mkdtemp(), 'destination') + shutil.copytree(src_dir, dst_dir, ignore=_ignore) + self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', + 'test.txt'))) + + dst_dir = join(self.mkdtemp(), 'destination') + shutil.copytree(pathlib.Path(src_dir), dst_dir, ignore=_ignore) + self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', + 'test.txt'))) + + dst_dir = join(self.mkdtemp(), 'destination') + src_dir_entry = list(os.scandir(tmp_dir))[0] + self.assertIsInstance(src_dir_entry, os.DirEntry) + shutil.copytree(src_dir_entry, dst_dir, ignore=_ignore) + self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', + 'test.txt'))) + + self.assertEqual(len(invokations), 9) + + def test_copytree_retains_permissions(self): + tmp_dir = self.mkdtemp() + src_dir = os.path.join(tmp_dir, 'source') + os.mkdir(src_dir) + dst_dir = os.path.join(tmp_dir, 'destination') + self.addCleanup(shutil.rmtree, tmp_dir) + + os.chmod(src_dir, 0o777) + write_file((src_dir, 'permissive.txt'), '123') + os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777) + write_file((src_dir, 'restrictive.txt'), '456') + os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600) + restrictive_subdir = tempfile.mkdtemp(dir=src_dir) + self.addCleanup(os_helper.rmtree, restrictive_subdir) + os.chmod(restrictive_subdir, 0o600) + + shutil.copytree(src_dir, dst_dir) + self.assertEqual(os.stat(src_dir).st_mode, os.stat(dst_dir).st_mode) + self.assertEqual(os.stat(os.path.join(src_dir, 'permissive.txt')).st_mode, + os.stat(os.path.join(dst_dir, 'permissive.txt')).st_mode) + self.assertEqual(os.stat(os.path.join(src_dir, 'restrictive.txt')).st_mode, + os.stat(os.path.join(dst_dir, 'restrictive.txt')).st_mode) + restrictive_subdir_dst = os.path.join(dst_dir, + os.path.split(restrictive_subdir)[1]) + self.assertEqual(os.stat(restrictive_subdir).st_mode, + os.stat(restrictive_subdir_dst).st_mode) + + @unittest.mock.patch('os.chmod') + def test_copytree_winerror(self, mock_patch): + # When copying to VFAT, copystat() raises OSError. On Windows, the + # exception object has a meaningful 'winerror' attribute, but not + # on other operating systems. Do not assume 'winerror' is set. + src_dir = self.mkdtemp() + dst_dir = os.path.join(self.mkdtemp(), 'destination') + self.addCleanup(shutil.rmtree, src_dir) + self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) + + mock_patch.side_effect = PermissionError('ka-boom') + with self.assertRaises(shutil.Error): + shutil.copytree(src_dir, dst_dir) + + def test_copytree_custom_copy_function(self): + # See: https://bugs.python.org/issue35648 + def custom_cpfun(a, b): + flag.append(None) + self.assertIsInstance(a, str) + self.assertIsInstance(b, str) + self.assertEqual(a, os.path.join(src, 'foo')) + self.assertEqual(b, os.path.join(dst, 'foo')) + + flag = [] + src = self.mkdtemp() + dst = tempfile.mktemp(dir=self.mkdtemp()) + with open(os.path.join(src, 'foo'), 'w', encoding='utf-8') as f: + f.close() + shutil.copytree(src, dst, copy_function=custom_cpfun) + self.assertEqual(len(flag), 1) + + # Issue #3002: copyfile and copytree block indefinitely on named pipes + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @os_helper.skip_unless_symlink + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_copytree_named_pipe(self): + os.mkdir(TESTFN) + try: + subdir = os.path.join(TESTFN, "subdir") + os.mkdir(subdir) + pipe = os.path.join(subdir, "mypipe") + try: + os.mkfifo(pipe) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) + try: + shutil.copytree(TESTFN, TESTFN2) + except shutil.Error as e: + errors = e.args[0] + self.assertEqual(len(errors), 1) + src, dst, error_msg = errors[0] + self.assertEqual("`%s` is a named pipe" % pipe, error_msg) + else: + self.fail("shutil.Error should have been raised") + finally: + shutil.rmtree(TESTFN, ignore_errors=True) + shutil.rmtree(TESTFN2, ignore_errors=True) + + def test_copytree_special_func(self): + src_dir = self.mkdtemp() + dst_dir = os.path.join(self.mkdtemp(), 'destination') + write_file((src_dir, 'test.txt'), '123') + os.mkdir(os.path.join(src_dir, 'test_dir')) + write_file((src_dir, 'test_dir', 'test.txt'), '456') + + copied = [] + def _copy(src, dst): + copied.append((src, dst)) + + shutil.copytree(src_dir, dst_dir, copy_function=_copy) + self.assertEqual(len(copied), 2) + + @os_helper.skip_unless_symlink + def test_copytree_dangling_symlinks(self): + src_dir = self.mkdtemp() + valid_file = os.path.join(src_dir, 'test.txt') + write_file(valid_file, 'abc') + dir_a = os.path.join(src_dir, 'dir_a') + os.mkdir(dir_a) + for d in src_dir, dir_a: + os.symlink('IDONTEXIST', os.path.join(d, 'broken')) + os.symlink(valid_file, os.path.join(d, 'valid')) + + # A dangling symlink should raise an error. + dst_dir = os.path.join(self.mkdtemp(), 'destination') + self.assertRaises(Error, shutil.copytree, src_dir, dst_dir) + + # Dangling symlinks should be ignored with the proper flag. + dst_dir = os.path.join(self.mkdtemp(), 'destination2') + shutil.copytree(src_dir, dst_dir, ignore_dangling_symlinks=True) + for root, dirs, files in os.walk(dst_dir): + self.assertNotIn('broken', files) + self.assertIn('valid', files) + + # a dangling symlink is copied if symlinks=True + dst_dir = os.path.join(self.mkdtemp(), 'destination3') + shutil.copytree(src_dir, dst_dir, symlinks=True) + self.assertIn('test.txt', os.listdir(dst_dir)) + + @os_helper.skip_unless_symlink + def test_copytree_symlink_dir(self): + src_dir = self.mkdtemp() + dst_dir = os.path.join(self.mkdtemp(), 'destination') + os.mkdir(os.path.join(src_dir, 'real_dir')) + with open(os.path.join(src_dir, 'real_dir', 'test.txt'), 'wb'): + pass + os.symlink(os.path.join(src_dir, 'real_dir'), + os.path.join(src_dir, 'link_to_dir'), + target_is_directory=True) + + shutil.copytree(src_dir, dst_dir, symlinks=False) + self.assertFalse(os.path.islink(os.path.join(dst_dir, 'link_to_dir'))) + self.assertIn('test.txt', os.listdir(os.path.join(dst_dir, 'link_to_dir'))) + + dst_dir = os.path.join(self.mkdtemp(), 'destination2') + shutil.copytree(src_dir, dst_dir, symlinks=True) + self.assertTrue(os.path.islink(os.path.join(dst_dir, 'link_to_dir'))) + self.assertIn('test.txt', os.listdir(os.path.join(dst_dir, 'link_to_dir'))) + + def test_copytree_return_value(self): + # copytree returns its destination path. + src_dir = self.mkdtemp() + dst_dir = src_dir + "dest" + self.addCleanup(shutil.rmtree, dst_dir, True) + src = os.path.join(src_dir, 'foo') + write_file(src, 'foo') + rv = shutil.copytree(src_dir, dst_dir) + self.assertEqual(['foo'], os.listdir(rv)) + + def test_copytree_subdirectory(self): + # copytree where dst is a subdirectory of src, see Issue 38688 + base_dir = self.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir, ignore_errors=True) + src_dir = os.path.join(base_dir, "t", "pg") + dst_dir = os.path.join(src_dir, "somevendor", "1.0") + os.makedirs(src_dir) + src = os.path.join(src_dir, 'pol') + write_file(src, 'pol') + rv = shutil.copytree(src_dir, dst_dir) + self.assertEqual(['pol'], os.listdir(rv)) + +class TestCopy(BaseTest, unittest.TestCase): + + ### shutil.copymode + + @os_helper.skip_unless_symlink + def test_copymode_follow_symlinks(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + dst_link = os.path.join(tmp_dir, 'quux') + write_file(src, 'foo') + write_file(dst, 'foo') + os.symlink(src, src_link) + os.symlink(dst, dst_link) + os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) + # file to file + os.chmod(dst, stat.S_IRWXO) + self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + shutil.copymode(src, dst) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # On Windows, os.chmod does not follow symlinks (issue #15411) + # follow src link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow dst link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow both links + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @unittest.skipUnless(hasattr(os, 'lchmod') or os.name == 'nt', 'requires os.lchmod') + @os_helper.skip_unless_symlink + def test_copymode_symlink_to_symlink(self): + _lchmod = os.chmod if os.name == 'nt' else os.lchmod + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + dst_link = os.path.join(tmp_dir, 'quux') + write_file(src, 'foo') + write_file(dst, 'foo') + os.symlink(src, src_link) + os.symlink(dst, dst_link) + os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) + os.chmod(dst, stat.S_IRWXU) + _lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) + # link to link + _lchmod(dst_link, stat.S_IRWXO) + old_mode = os.stat(dst).st_mode + shutil.copymode(src_link, dst_link, follow_symlinks=False) + self.assertEqual(os.lstat(src_link).st_mode, + os.lstat(dst_link).st_mode) + self.assertEqual(os.stat(dst).st_mode, old_mode) + # src link - use chmod + _lchmod(dst_link, stat.S_IRWXO) + shutil.copymode(src_link, dst, follow_symlinks=False) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # dst link - use chmod + _lchmod(dst_link, stat.S_IRWXO) + shutil.copymode(src, dst_link, follow_symlinks=False) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + + @unittest.skipIf(hasattr(os, 'lchmod'), 'requires os.lchmod to be missing') + @os_helper.skip_unless_symlink + def test_copymode_symlink_to_symlink_wo_lchmod(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + dst_link = os.path.join(tmp_dir, 'quux') + write_file(src, 'foo') + write_file(dst, 'foo') + os.symlink(src, src_link) + os.symlink(dst, dst_link) + shutil.copymode(src_link, dst_link, follow_symlinks=False) # silent fail + + ### shutil.copystat + + @os_helper.skip_unless_symlink + def test_copystat_symlinks(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + dst_link = os.path.join(tmp_dir, 'qux') + write_file(src, 'foo') + src_stat = os.stat(src) + os.utime(src, (src_stat.st_atime, + src_stat.st_mtime - 42.0)) # ensure different mtimes + write_file(dst, 'bar') + self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime) + os.symlink(src, src_link) + os.symlink(dst, dst_link) + if hasattr(os, 'lchmod'): + os.lchmod(src_link, stat.S_IRWXO) + elif os.name == 'nt': + os.chmod(src_link, stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(src_link, stat.UF_NODUMP) + src_link_stat = os.lstat(src_link) + # follow + if hasattr(os, 'lchmod') or os.name == 'nt': + shutil.copystat(src_link, dst_link, follow_symlinks=True) + self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode) + # don't follow + shutil.copystat(src_link, dst_link, follow_symlinks=False) + dst_link_stat = os.lstat(dst_link) + if os.utime in os.supports_follow_symlinks: + for attr in 'st_atime', 'st_mtime': + # The modification times may be truncated in the new file. + self.assertLessEqual(getattr(src_link_stat, attr), + getattr(dst_link_stat, attr) + 1) + if hasattr(os, 'lchmod') or os.name == 'nt': self.assertEqual(src_link_stat.st_mode, dst_link_stat.st_mode) if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'): self.assertEqual(src_link_stat.st_flags, dst_link_stat.st_flags) @@ -534,6 +1202,8 @@ def _chflags_raiser(path, flags, *, follow_symlinks=True): finally: os.chflags = old_chflags + ### shutil.copyxattr + @os_helper.skip_unless_xattr def test_copyxattr(self): tmp_dir = self.mkdtemp() @@ -600,8 +1270,7 @@ def _raise_on_src(fname, *, follow_symlinks=True): @os_helper.skip_unless_symlink @os_helper.skip_unless_xattr - @unittest.skipUnless(hasattr(os, 'geteuid') and os.geteuid() == 0, - 'root privileges required') + @os_helper.skip_unless_dac_override def test_copyxattr_symlinks(self): # On Linux, it's only possible to access non-user xattr for symlinks; # which in turn require root privileges. This test should be expanded @@ -623,31 +1292,26 @@ def test_copyxattr_symlinks(self): shutil._copyxattr(src_link, dst, follow_symlinks=False) self.assertEqual(os.getxattr(dst, 'trusted.foo'), b'43') - @os_helper.skip_unless_symlink - def test_copy_symlinks(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') - os.symlink(src, src_link) - if hasattr(os, 'lchmod'): - os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) - # don't follow - shutil.copy(src_link, dst, follow_symlinks=True) - self.assertFalse(os.path.islink(dst)) - self.assertEqual(read_file(src), read_file(dst)) - os.remove(dst) - # follow - shutil.copy(src_link, dst, follow_symlinks=False) - self.assertTrue(os.path.islink(dst)) - self.assertEqual(os.readlink(dst), os.readlink(src_link)) - if hasattr(os, 'lchmod'): - self.assertEqual(os.lstat(src_link).st_mode, - os.lstat(dst).st_mode) + ### shutil.copy + + def _copy_file(self, method): + fname = 'test.txt' + tmpdir = self.mkdtemp() + write_file((tmpdir, fname), 'xxx') + file1 = os.path.join(tmpdir, fname) + tmpdir2 = self.mkdtemp() + method(file1, tmpdir2) + file2 = os.path.join(tmpdir2, fname) + return (file1, file2) + + def test_copy(self): + # Ensure that the copied file exists and has the same mode bits. + file1, file2 = self._copy_file(shutil.copy) + self.assertTrue(os.path.exists(file2)) + self.assertEqual(os.stat(file1).st_mode, os.stat(file2).st_mode) @os_helper.skip_unless_symlink - def test_copy2_symlinks(self): + def test_copy_symlinks(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') @@ -656,333 +1320,142 @@ def test_copy2_symlinks(self): os.symlink(src, src_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) - if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): - os.lchflags(src_link, stat.UF_NODUMP) - src_stat = os.stat(src) - src_link_stat = os.lstat(src_link) - # follow - shutil.copy2(src_link, dst, follow_symlinks=True) + # don't follow + shutil.copy(src_link, dst, follow_symlinks=True) self.assertFalse(os.path.islink(dst)) self.assertEqual(read_file(src), read_file(dst)) os.remove(dst) - # don't follow - shutil.copy2(src_link, dst, follow_symlinks=False) + # follow + shutil.copy(src_link, dst, follow_symlinks=False) self.assertTrue(os.path.islink(dst)) self.assertEqual(os.readlink(dst), os.readlink(src_link)) - dst_stat = os.lstat(dst) - if os.utime in os.supports_follow_symlinks: - for attr in 'st_atime', 'st_mtime': - # The modification times may be truncated in the new file. - self.assertLessEqual(getattr(src_link_stat, attr), - getattr(dst_stat, attr) + 1) - if hasattr(os, 'lchmod'): - self.assertEqual(src_link_stat.st_mode, dst_stat.st_mode) - self.assertNotEqual(src_stat.st_mode, dst_stat.st_mode) - if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'): - self.assertEqual(src_link_stat.st_flags, dst_stat.st_flags) - - @os_helper.skip_unless_xattr - def test_copy2_xattr(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - write_file(src, 'foo') - os.setxattr(src, 'user.foo', b'42') - shutil.copy2(src, dst) - self.assertEqual( - os.getxattr(src, 'user.foo'), - os.getxattr(dst, 'user.foo')) - os.remove(dst) - - @os_helper.skip_unless_symlink - def test_copyfile_symlinks(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'src') - dst = os.path.join(tmp_dir, 'dst') - dst_link = os.path.join(tmp_dir, 'dst_link') - link = os.path.join(tmp_dir, 'link') - write_file(src, 'foo') - os.symlink(src, link) - # don't follow - shutil.copyfile(link, dst_link, follow_symlinks=False) - self.assertTrue(os.path.islink(dst_link)) - self.assertEqual(os.readlink(link), os.readlink(dst_link)) - # follow - shutil.copyfile(link, dst) - self.assertFalse(os.path.islink(dst)) - - def test_rmtree_uses_safe_fd_version_if_available(self): - _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= - os.supports_dir_fd and - os.listdir in os.supports_fd and - os.stat in os.supports_follow_symlinks) - if _use_fd_functions: - self.assertTrue(shutil._use_fd_functions) - self.assertTrue(shutil.rmtree.avoids_symlink_attacks) - tmp_dir = self.mkdtemp() - d = os.path.join(tmp_dir, 'a') - os.mkdir(d) - try: - real_rmtree = shutil._rmtree_safe_fd - class Called(Exception): pass - def _raiser(*args, **kwargs): - raise Called - shutil._rmtree_safe_fd = _raiser - self.assertRaises(Called, shutil.rmtree, d) - finally: - shutil._rmtree_safe_fd = real_rmtree - else: - self.assertFalse(shutil._use_fd_functions) - self.assertFalse(shutil.rmtree.avoids_symlink_attacks) - - def test_rmtree_dont_delete_file(self): - # When called on a file instead of a directory, don't delete it. - handle, path = tempfile.mkstemp() - os.close(handle) - self.assertRaises(NotADirectoryError, shutil.rmtree, path) - os.remove(path) - - def test_copytree_simple(self): - src_dir = tempfile.mkdtemp() - dst_dir = os.path.join(tempfile.mkdtemp(), 'destination') - self.addCleanup(shutil.rmtree, src_dir) - self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) - write_file((src_dir, 'test.txt'), '123') - os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') - - shutil.copytree(src_dir, dst_dir) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) - self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir'))) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir', - 'test.txt'))) - actual = read_file((dst_dir, 'test.txt')) - self.assertEqual(actual, '123') - actual = read_file((dst_dir, 'test_dir', 'test.txt')) - self.assertEqual(actual, '456') - - def test_copytree_dirs_exist_ok(self): - src_dir = tempfile.mkdtemp() - dst_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, src_dir) - self.addCleanup(shutil.rmtree, dst_dir) - - write_file((src_dir, 'nonexisting.txt'), '123') - os.mkdir(os.path.join(src_dir, 'existing_dir')) - os.mkdir(os.path.join(dst_dir, 'existing_dir')) - write_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') - write_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') - - shutil.copytree(src_dir, dst_dir, dirs_exist_ok=True) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'nonexisting.txt'))) - self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'existing_dir'))) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'existing_dir', - 'existing.txt'))) - actual = read_file((dst_dir, 'nonexisting.txt')) - self.assertEqual(actual, '123') - actual = read_file((dst_dir, 'existing_dir', 'existing.txt')) - self.assertEqual(actual, 'has been replaced') - - with self.assertRaises(FileExistsError): - shutil.copytree(src_dir, dst_dir, dirs_exist_ok=False) - - @os_helper.skip_unless_symlink - def test_copytree_symlinks(self): - tmp_dir = self.mkdtemp() - src_dir = os.path.join(tmp_dir, 'src') - dst_dir = os.path.join(tmp_dir, 'dst') - sub_dir = os.path.join(src_dir, 'sub') - os.mkdir(src_dir) - os.mkdir(sub_dir) - write_file((src_dir, 'file.txt'), 'foo') - src_link = os.path.join(sub_dir, 'link') - dst_link = os.path.join(dst_dir, 'sub/link') - os.symlink(os.path.join(src_dir, 'file.txt'), - src_link) - if hasattr(os, 'lchmod'): - os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) - if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): - os.lchflags(src_link, stat.UF_NODUMP) - src_stat = os.lstat(src_link) - shutil.copytree(src_dir, dst_dir, symlinks=True) - self.assertTrue(os.path.islink(os.path.join(dst_dir, 'sub', 'link'))) - actual = os.readlink(os.path.join(dst_dir, 'sub', 'link')) - # Bad practice to blindly strip the prefix as it may be required to - # correctly refer to the file, but we're only comparing paths here. - if os.name == 'nt' and actual.startswith('\\\\?\\'): - actual = actual[4:] - self.assertEqual(actual, os.path.join(src_dir, 'file.txt')) - dst_stat = os.lstat(dst_link) if hasattr(os, 'lchmod'): - self.assertEqual(dst_stat.st_mode, src_stat.st_mode) - if hasattr(os, 'lchflags'): - self.assertEqual(dst_stat.st_flags, src_stat.st_flags) - - def test_copytree_with_exclude(self): - # creating data - join = os.path.join - exists = os.path.exists - src_dir = tempfile.mkdtemp() - try: - dst_dir = join(tempfile.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') - write_file((src_dir, 'test.tmp'), '123') - os.mkdir(join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') - os.mkdir(join(src_dir, 'test_dir2')) - write_file((src_dir, 'test_dir2', 'test.txt'), '456') - os.mkdir(join(src_dir, 'test_dir2', 'subdir')) - os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) - write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') - write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') - - # testing glob-like patterns - try: - patterns = shutil.ignore_patterns('*.tmp', 'test_dir2') - shutil.copytree(src_dir, dst_dir, ignore=patterns) - # checking the result: some elements should not be copied - self.assertTrue(exists(join(dst_dir, 'test.txt'))) - self.assertFalse(exists(join(dst_dir, 'test.tmp'))) - self.assertFalse(exists(join(dst_dir, 'test_dir2'))) - finally: - shutil.rmtree(dst_dir) - try: - patterns = shutil.ignore_patterns('*.tmp', 'subdir*') - shutil.copytree(src_dir, dst_dir, ignore=patterns) - # checking the result: some elements should not be copied - self.assertFalse(exists(join(dst_dir, 'test.tmp'))) - self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2'))) - self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir'))) - finally: - shutil.rmtree(dst_dir) - - # testing callable-style - try: - def _filter(src, names): - res = [] - for name in names: - path = os.path.join(src, name) - - if (os.path.isdir(path) and - path.split()[-1] == 'subdir'): - res.append(name) - elif os.path.splitext(path)[-1] in ('.py'): - res.append(name) - return res - - shutil.copytree(src_dir, dst_dir, ignore=_filter) - - # checking the result: some elements should not be copied - self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2', - 'test.py'))) - self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir'))) - - finally: - shutil.rmtree(dst_dir) - finally: - shutil.rmtree(src_dir) - shutil.rmtree(os.path.dirname(dst_dir)) - - def test_copytree_arg_types_of_ignore(self): - join = os.path.join - exists = os.path.exists - - tmp_dir = self.mkdtemp() - src_dir = join(tmp_dir, "source") - - os.mkdir(join(src_dir)) - os.mkdir(join(src_dir, 'test_dir')) - os.mkdir(os.path.join(src_dir, 'test_dir', 'subdir')) - write_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') - - invokations = [] - - def _ignore(src, names): - invokations.append(src) - self.assertIsInstance(src, str) - self.assertIsInstance(names, list) - self.assertEqual(len(names), len(set(names))) - for name in names: - self.assertIsInstance(name, str) - return [] + self.assertEqual(os.lstat(src_link).st_mode, + os.lstat(dst).st_mode) - dst_dir = join(self.mkdtemp(), 'destination') - shutil.copytree(src_dir, dst_dir, ignore=_ignore) - self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', - 'test.txt'))) + ### shutil.copy2 - dst_dir = join(self.mkdtemp(), 'destination') - shutil.copytree(pathlib.Path(src_dir), dst_dir, ignore=_ignore) - self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', - 'test.txt'))) + @unittest.skipUnless(hasattr(os, 'utime'), 'requires os.utime') + def test_copy2(self): + # Ensure that the copied file exists and has the same mode and + # modification time bits. + file1, file2 = self._copy_file(shutil.copy2) + self.assertTrue(os.path.exists(file2)) + file1_stat = os.stat(file1) + file2_stat = os.stat(file2) + self.assertEqual(file1_stat.st_mode, file2_stat.st_mode) + for attr in 'st_atime', 'st_mtime': + # The modification times may be truncated in the new file. + self.assertLessEqual(getattr(file1_stat, attr), + getattr(file2_stat, attr) + 1) + if hasattr(os, 'chflags') and hasattr(file1_stat, 'st_flags'): + self.assertEqual(getattr(file1_stat, 'st_flags'), + getattr(file2_stat, 'st_flags')) - dst_dir = join(self.mkdtemp(), 'destination') - src_dir_entry = list(os.scandir(tmp_dir))[0] - self.assertIsInstance(src_dir_entry, os.DirEntry) - shutil.copytree(src_dir_entry, dst_dir, ignore=_ignore) - self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', - 'test.txt'))) + @os_helper.skip_unless_symlink + def test_copy2_symlinks(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + write_file(src, 'foo') + os.symlink(src, src_link) + if hasattr(os, 'lchmod'): + os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(src_link, stat.UF_NODUMP) + src_stat = os.stat(src) + src_link_stat = os.lstat(src_link) + # follow + shutil.copy2(src_link, dst, follow_symlinks=True) + self.assertFalse(os.path.islink(dst)) + self.assertEqual(read_file(src), read_file(dst)) + os.remove(dst) + # don't follow + shutil.copy2(src_link, dst, follow_symlinks=False) + self.assertTrue(os.path.islink(dst)) + self.assertEqual(os.readlink(dst), os.readlink(src_link)) + dst_stat = os.lstat(dst) + if os.utime in os.supports_follow_symlinks: + for attr in 'st_atime', 'st_mtime': + # The modification times may be truncated in the new file. + self.assertLessEqual(getattr(src_link_stat, attr), + getattr(dst_stat, attr) + 1) + if hasattr(os, 'lchmod'): + self.assertEqual(src_link_stat.st_mode, dst_stat.st_mode) + self.assertNotEqual(src_stat.st_mode, dst_stat.st_mode) + if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'): + self.assertEqual(src_link_stat.st_flags, dst_stat.st_flags) - self.assertEqual(len(invokations), 9) + @os_helper.skip_unless_xattr + def test_copy2_xattr(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + write_file(src, 'foo') + os.setxattr(src, 'user.foo', b'42') + shutil.copy2(src, dst) + self.assertEqual( + os.getxattr(src, 'user.foo'), + os.getxattr(dst, 'user.foo')) + os.remove(dst) - def test_copytree_retains_permissions(self): - tmp_dir = tempfile.mkdtemp() - src_dir = os.path.join(tmp_dir, 'source') - os.mkdir(src_dir) - dst_dir = os.path.join(tmp_dir, 'destination') - self.addCleanup(shutil.rmtree, tmp_dir) + def test_copy_return_value(self): + # copy and copy2 both return their destination path. + for fn in (shutil.copy, shutil.copy2): + src_dir = self.mkdtemp() + dst_dir = self.mkdtemp() + src = os.path.join(src_dir, 'foo') + write_file(src, 'foo') + rv = fn(src, dst_dir) + self.assertEqual(rv, os.path.join(dst_dir, 'foo')) + rv = fn(src, os.path.join(dst_dir, 'bar')) + self.assertEqual(rv, os.path.join(dst_dir, 'bar')) - os.chmod(src_dir, 0o777) - write_file((src_dir, 'permissive.txt'), '123') - os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777) - write_file((src_dir, 'restrictive.txt'), '456') - os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600) - restrictive_subdir = tempfile.mkdtemp(dir=src_dir) - os.chmod(restrictive_subdir, 0o600) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_copy_dir(self): + self._test_copy_dir(shutil.copy) - shutil.copytree(src_dir, dst_dir) - self.assertEqual(os.stat(src_dir).st_mode, os.stat(dst_dir).st_mode) - self.assertEqual(os.stat(os.path.join(src_dir, 'permissive.txt')).st_mode, - os.stat(os.path.join(dst_dir, 'permissive.txt')).st_mode) - self.assertEqual(os.stat(os.path.join(src_dir, 'restrictive.txt')).st_mode, - os.stat(os.path.join(dst_dir, 'restrictive.txt')).st_mode) - restrictive_subdir_dst = os.path.join(dst_dir, - os.path.split(restrictive_subdir)[1]) - self.assertEqual(os.stat(restrictive_subdir).st_mode, - os.stat(restrictive_subdir_dst).st_mode) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_copy2_dir(self): + self._test_copy_dir(shutil.copy2) - @unittest.mock.patch('os.chmod') - def test_copytree_winerror(self, mock_patch): - # When copying to VFAT, copystat() raises OSError. On Windows, the - # exception object has a meaningful 'winerror' attribute, but not - # on other operating systems. Do not assume 'winerror' is set. - src_dir = tempfile.mkdtemp() - dst_dir = os.path.join(tempfile.mkdtemp(), 'destination') - self.addCleanup(shutil.rmtree, src_dir) - self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) + def _test_copy_dir(self, copy_func): + src_dir = self.mkdtemp() + src_file = os.path.join(src_dir, 'foo') + dir2 = self.mkdtemp() + dst = os.path.join(src_dir, 'does_not_exist/') + write_file(src_file, 'foo') + if sys.platform == "win32": + err = PermissionError + else: + err = IsADirectoryError + self.assertRaises(err, copy_func, dir2, src_dir) - mock_patch.side_effect = PermissionError('ka-boom') - with self.assertRaises(shutil.Error): - shutil.copytree(src_dir, dst_dir) + # raise *err* because of src rather than FileNotFoundError because of dst + self.assertRaises(err, copy_func, dir2, dst) + copy_func(src_file, dir2) # should not raise exceptions - def test_copytree_custom_copy_function(self): - # See: https://bugs.python.org/issue35648 - def custom_cpfun(a, b): - flag.append(None) - self.assertIsInstance(a, str) - self.assertIsInstance(b, str) - self.assertEqual(a, os.path.join(src, 'foo')) - self.assertEqual(b, os.path.join(dst, 'foo')) + ### shutil.copyfile - flag = [] - src = tempfile.mkdtemp() - self.addCleanup(os_helper.rmtree, src) - dst = tempfile.mktemp() - self.addCleanup(os_helper.rmtree, dst) - with open(os.path.join(src, 'foo'), 'w') as f: - f.close() - shutil.copytree(src, dst, copy_function=custom_cpfun) - self.assertEqual(len(flag), 1) + @os_helper.skip_unless_symlink + def test_copyfile_symlinks(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'src') + dst = os.path.join(tmp_dir, 'dst') + dst_link = os.path.join(tmp_dir, 'dst_link') + link = os.path.join(tmp_dir, 'link') + write_file(src, 'foo') + os.symlink(src, link) + # don't follow + shutil.copyfile(link, dst_link, follow_symlinks=False) + self.assertTrue(os.path.islink(dst_link)) + self.assertEqual(os.readlink(link), os.readlink(dst_link)) + # follow + shutil.copyfile(link, dst) + self.assertFalse(os.path.islink(dst)) @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') def test_dont_copy_file_onto_link_to_itself(self): @@ -991,14 +1464,14 @@ def test_dont_copy_file_onto_link_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w') as f: + with open(src, 'w', encoding='utf-8') as f: f.write('cheddar') try: os.link(src, dst) except PermissionError as e: self.skipTest('os.link(): %s' % e) self.assertRaises(shutil.SameFileError, shutil.copyfile, src, dst) - with open(src, 'r') as f: + with open(src, 'r', encoding='utf-8') as f: self.assertEqual(f.read(), 'cheddar') os.remove(dst) finally: @@ -1011,49 +1484,23 @@ def test_dont_copy_file_onto_symlink_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w') as f: + with open(src, 'w', encoding='utf-8') as f: f.write('cheddar') # Using `src` here would mean we end up with a symlink pointing # to TESTFN/TESTFN/cheese, while it should point at # TESTFN/cheese. os.symlink('cheese', dst) self.assertRaises(shutil.SameFileError, shutil.copyfile, src, dst) - with open(src, 'r') as f: + with open(src, 'r', encoding='utf-8') as f: self.assertEqual(f.read(), 'cheddar') os.remove(dst) finally: shutil.rmtree(TESTFN, ignore_errors=True) - @os_helper.skip_unless_symlink - def test_rmtree_on_symlink(self): - # bug 1669. - os.mkdir(TESTFN) - try: - src = os.path.join(TESTFN, 'cheese') - dst = os.path.join(TESTFN, 'shop') - os.mkdir(src) - os.symlink(src, dst) - self.assertRaises(OSError, shutil.rmtree, dst) - shutil.rmtree(dst, ignore_errors=True) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - @unittest.skipUnless(_winapi, 'only relevant on Windows') - def test_rmtree_on_junction(self): - os.mkdir(TESTFN) - try: - src = os.path.join(TESTFN, 'cheese') - dst = os.path.join(TESTFN, 'shop') - os.mkdir(src) - open(os.path.join(src, 'spam'), 'wb').close() - _winapi.CreateJunction(src, dst) - self.assertRaises(OSError, shutil.rmtree, dst) - shutil.rmtree(dst, ignore_errors=True) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - # Issue #3002: copyfile and copytree block indefinitely on named pipes @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") def test_copyfile_named_pipe(self): try: os.mkfifo(TESTFN) @@ -1067,121 +1514,68 @@ def test_copyfile_named_pipe(self): finally: os.remove(TESTFN) - @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') - @os_helper.skip_unless_symlink - def test_copytree_named_pipe(self): - os.mkdir(TESTFN) - try: - subdir = os.path.join(TESTFN, "subdir") - os.mkdir(subdir) - pipe = os.path.join(subdir, "mypipe") - try: - os.mkfifo(pipe) - except PermissionError as e: - self.skipTest('os.mkfifo(): %s' % e) - try: - shutil.copytree(TESTFN, TESTFN2) - except shutil.Error as e: - errors = e.args[0] - self.assertEqual(len(errors), 1) - src, dst, error_msg = errors[0] - self.assertEqual("`%s` is a named pipe" % pipe, error_msg) - else: - self.fail("shutil.Error should have been raised") - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - shutil.rmtree(TESTFN2, ignore_errors=True) - - def test_copytree_special_func(self): - - src_dir = self.mkdtemp() - dst_dir = os.path.join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') - os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') - - copied = [] - def _copy(src, dst): - copied.append((src, dst)) - - shutil.copytree(src_dir, dst_dir, copy_function=_copy) - self.assertEqual(len(copied), 2) - - @os_helper.skip_unless_symlink - def test_copytree_dangling_symlinks(self): - - # a dangling symlink raises an error at the end + def test_copyfile_return_value(self): + # copytree returns its destination path. src_dir = self.mkdtemp() - dst_dir = os.path.join(self.mkdtemp(), 'destination') - os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt')) - os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') - self.assertRaises(Error, shutil.copytree, src_dir, dst_dir) - - # a dangling symlink is ignored with the proper flag - dst_dir = os.path.join(self.mkdtemp(), 'destination2') - shutil.copytree(src_dir, dst_dir, ignore_dangling_symlinks=True) - self.assertNotIn('test.txt', os.listdir(dst_dir)) - - # a dangling symlink is copied if symlinks=True - dst_dir = os.path.join(self.mkdtemp(), 'destination3') - shutil.copytree(src_dir, dst_dir, symlinks=True) - self.assertIn('test.txt', os.listdir(dst_dir)) + dst_dir = self.mkdtemp() + dst_file = os.path.join(dst_dir, 'bar') + src_file = os.path.join(src_dir, 'foo') + write_file(src_file, 'foo') + rv = shutil.copyfile(src_file, dst_file) + self.assertTrue(os.path.exists(rv)) + self.assertEqual(read_file(src_file), read_file(dst_file)) - @os_helper.skip_unless_symlink - def test_copytree_symlink_dir(self): + def test_copyfile_same_file(self): + # copyfile() should raise SameFileError if the source and destination + # are the same. src_dir = self.mkdtemp() - dst_dir = os.path.join(self.mkdtemp(), 'destination') - os.mkdir(os.path.join(src_dir, 'real_dir')) - with open(os.path.join(src_dir, 'real_dir', 'test.txt'), 'w'): - pass - os.symlink(os.path.join(src_dir, 'real_dir'), - os.path.join(src_dir, 'link_to_dir'), - target_is_directory=True) + src_file = os.path.join(src_dir, 'foo') + write_file(src_file, 'foo') + self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file) + # But Error should work too, to stay backward compatible. + self.assertRaises(Error, shutil.copyfile, src_file, src_file) + # Make sure file is not corrupted. + self.assertEqual(read_file(src_file), 'foo') - shutil.copytree(src_dir, dst_dir, symlinks=False) - self.assertFalse(os.path.islink(os.path.join(dst_dir, 'link_to_dir'))) - self.assertIn('test.txt', os.listdir(os.path.join(dst_dir, 'link_to_dir'))) + @unittest.skipIf(MACOS or SOLARIS or _winapi, 'On MACOS, Solaris and Windows the errors are not confusing (though different)') + # gh-92670: The test uses a trailing slash to force the OS consider + # the path as a directory, but on AIX the trailing slash has no effect + # and is considered as a file. + @unittest.skipIf(AIX, 'Not valid on AIX, see gh-92670') + def test_copyfile_nonexistent_dir(self): + # Issue 43219 + src_dir = self.mkdtemp() + src_file = os.path.join(src_dir, 'foo') + dst = os.path.join(src_dir, 'does_not_exist/') + write_file(src_file, 'foo') + self.assertRaises(FileNotFoundError, shutil.copyfile, src_file, dst) - dst_dir = os.path.join(self.mkdtemp(), 'destination2') - shutil.copytree(src_dir, dst_dir, symlinks=True) - self.assertTrue(os.path.islink(os.path.join(dst_dir, 'link_to_dir'))) - self.assertIn('test.txt', os.listdir(os.path.join(dst_dir, 'link_to_dir'))) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_copyfile_copy_dir(self): + # Issue 45234 + # test copy() and copyfile() raising proper exceptions when src and/or + # dst are directories + src_dir = self.mkdtemp() + src_file = os.path.join(src_dir, 'foo') + dir2 = self.mkdtemp() + dst = os.path.join(src_dir, 'does_not_exist/') + write_file(src_file, 'foo') + if sys.platform == "win32": + err = PermissionError + else: + err = IsADirectoryError - def _copy_file(self, method): - fname = 'test.txt' - tmpdir = self.mkdtemp() - write_file((tmpdir, fname), 'xxx') - file1 = os.path.join(tmpdir, fname) - tmpdir2 = self.mkdtemp() - method(file1, tmpdir2) - file2 = os.path.join(tmpdir2, fname) - return (file1, file2) + self.assertRaises(err, shutil.copyfile, src_dir, dst) + self.assertRaises(err, shutil.copyfile, src_file, src_dir) + self.assertRaises(err, shutil.copyfile, dir2, src_dir) - def test_copy(self): - # Ensure that the copied file exists and has the same mode bits. - file1, file2 = self._copy_file(shutil.copy) - self.assertTrue(os.path.exists(file2)) - self.assertEqual(os.stat(file1).st_mode, os.stat(file2).st_mode) - @unittest.skipUnless(hasattr(os, 'utime'), 'requires os.utime') - def test_copy2(self): - # Ensure that the copied file exists and has the same mode and - # modification time bits. - file1, file2 = self._copy_file(shutil.copy2) - self.assertTrue(os.path.exists(file2)) - file1_stat = os.stat(file1) - file2_stat = os.stat(file2) - self.assertEqual(file1_stat.st_mode, file2_stat.st_mode) - for attr in 'st_atime', 'st_mtime': - # The modification times may be truncated in the new file. - self.assertLessEqual(getattr(file1_stat, attr), - getattr(file2_stat, attr) + 1) - if hasattr(os, 'chflags') and hasattr(file1_stat, 'st_flags'): - self.assertEqual(getattr(file1_stat, 'st_flags'), - getattr(file2_stat, 'st_flags')) +class TestArchives(BaseTest, unittest.TestCase): + + ### shutil.make_archive - @support.requires_zlib + @support.requires_zlib() def test_make_tarball(self): # creating something to tar root_dir, base_dir = self._create_files('') @@ -1193,7 +1587,7 @@ def test_make_tarball(self): work_dir = os.path.dirname(tmpdir2) rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - with os_helper.change_cwd(work_dir): + with os_helper.change_cwd(work_dir), no_chdir: base_name = os.path.abspath(rel_base_name) tarball = make_archive(rel_base_name, 'gztar', root_dir, '.') @@ -1207,7 +1601,7 @@ def test_make_tarball(self): './file1', './file2', './sub/file3']) # trying an uncompressed one - with os_helper.change_cwd(work_dir): + with os_helper.change_cwd(work_dir), no_chdir: tarball = make_archive(rel_base_name, 'tar', root_dir, '.') self.assertEqual(tarball, base_name + '.tar') self.assertTrue(os.path.isfile(tarball)) @@ -1237,13 +1631,14 @@ def _create_files(self, base_dir='dist'): write_file((root_dir, 'outer'), 'xxx') return root_dir, base_dir - @support.requires_zlib + @support.requires_zlib() @unittest.skipUnless(shutil.which('tar'), 'Need the tar command to run') def test_tarfile_vs_tar(self): root_dir, base_dir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') - tarball = make_archive(base_name, 'gztar', root_dir, base_dir) + with no_chdir: + tarball = make_archive(base_name, 'gztar', root_dir, base_dir) # check if the compressed tarball was created self.assertEqual(tarball, base_name + '.tar.gz') @@ -1252,6 +1647,17 @@ def test_tarfile_vs_tar(self): # now create another tarball using `tar` tarball2 = os.path.join(root_dir, 'archive2.tar') tar_cmd = ['tar', '-cf', 'archive2.tar', base_dir] + if sys.platform == 'darwin': + # macOS tar can include extended attributes, + # ACLs and other mac specific metadata into the + # archive (an recentish version of the OS). + # + # This feature can be disabled with the + # '--no-mac-metadata' option on macOS 11 or + # later. + import platform + if int(platform.mac_ver()[0].split('.')[0]) >= 11: + tar_cmd.insert(1, '--no-mac-metadata') subprocess.check_call(tar_cmd, cwd=root_dir, stdout=subprocess.DEVNULL) @@ -1260,17 +1666,19 @@ def test_tarfile_vs_tar(self): self.assertEqual(self._tarinfo(tarball), self._tarinfo(tarball2)) # trying an uncompressed one - tarball = make_archive(base_name, 'tar', root_dir, base_dir) + with no_chdir: + tarball = make_archive(base_name, 'tar', root_dir, base_dir) self.assertEqual(tarball, base_name + '.tar') self.assertTrue(os.path.isfile(tarball)) # now for a dry_run - tarball = make_archive(base_name, 'tar', root_dir, base_dir, - dry_run=True) + with no_chdir: + tarball = make_archive(base_name, 'tar', root_dir, base_dir, + dry_run=True) self.assertEqual(tarball, base_name + '.tar') self.assertTrue(os.path.isfile(tarball)) - @support.requires_zlib + @support.requires_zlib() def test_make_zipfile(self): # creating something to zip root_dir, base_dir = self._create_files() @@ -1282,7 +1690,7 @@ def test_make_zipfile(self): work_dir = os.path.dirname(tmpdir2) rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - with os_helper.change_cwd(work_dir): + with os_helper.change_cwd(work_dir), no_chdir: base_name = os.path.abspath(rel_base_name) res = make_archive(rel_base_name, 'zip', root_dir) @@ -1295,7 +1703,7 @@ def test_make_zipfile(self): 'dist/file1', 'dist/file2', 'dist/sub/file3', 'outer']) - with os_helper.change_cwd(work_dir): + with os_helper.change_cwd(work_dir), no_chdir: base_name = os.path.abspath(rel_base_name) res = make_archive(rel_base_name, 'zip', root_dir, base_dir) @@ -1307,13 +1715,14 @@ def test_make_zipfile(self): ['dist/', 'dist/sub/', 'dist/sub2/', 'dist/file1', 'dist/file2', 'dist/sub/file3']) - @support.requires_zlib + @support.requires_zlib() @unittest.skipUnless(shutil.which('zip'), 'Need the zip command to run') def test_zipfile_vs_zip(self): root_dir, base_dir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') - archive = make_archive(base_name, 'zip', root_dir, base_dir) + with no_chdir: + archive = make_archive(base_name, 'zip', root_dir, base_dir) # check if ZIP file was created self.assertEqual(archive, base_name + '.zip') @@ -1333,13 +1742,14 @@ def test_zipfile_vs_zip(self): names2 = zf.namelist() self.assertEqual(sorted(names), sorted(names2)) - @support.requires_zlib + @support.requires_zlib() @unittest.skipUnless(shutil.which('unzip'), 'Need the unzip command to run') def test_unzip_zipfile(self): root_dir, base_dir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') - archive = make_archive(base_name, 'zip', root_dir, base_dir) + with no_chdir: + archive = make_archive(base_name, 'zip', root_dir, base_dir) # check if ZIP file was created self.assertEqual(archive, base_name + '.zip') @@ -1362,7 +1772,7 @@ def test_make_archive(self): base_name = os.path.join(tmpdir, 'archive') self.assertRaises(ValueError, make_archive, base_name, 'xxx') - @support.requires_zlib + @support.requires_zlib() def test_make_archive_owner_group(self): # testing make_archive with owner and group, with various combinations # this works even if there's not gid/uid support @@ -1390,14 +1800,14 @@ def test_make_archive_owner_group(self): self.assertTrue(os.path.isfile(res)) - @support.requires_zlib + @support.requires_zlib() @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") def test_tarfile_root_owner(self): root_dir, base_dir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') group = grp.getgrgid(0)[0] owner = pwd.getpwuid(0)[0] - with os_helper.change_cwd(root_dir): + with os_helper.change_cwd(root_dir), no_chdir: archive_name = make_archive(base_name, 'gztar', root_dir, 'dist', owner=owner, group=group) @@ -1413,17 +1823,61 @@ def test_tarfile_root_owner(self): finally: archive.close() + def test_make_archive_cwd_default(self): + current_dir = os.getcwd() + def archiver(base_name, base_dir, **kw): + self.assertNotIn('root_dir', kw) + self.assertEqual(base_name, 'basename') + self.assertEqual(os.getcwd(), current_dir) + raise RuntimeError() + + register_archive_format('xxx', archiver, [], 'xxx file') + try: + with no_chdir: + with self.assertRaises(RuntimeError): + make_archive('basename', 'xxx') + self.assertEqual(os.getcwd(), current_dir) + finally: + unregister_archive_format('xxx') + def test_make_archive_cwd(self): current_dir = os.getcwd() - def _breaks(*args, **kw): + root_dir = self.mkdtemp() + def archiver(base_name, base_dir, **kw): + self.assertNotIn('root_dir', kw) + self.assertEqual(base_name, os.path.join(current_dir, 'basename')) + self.assertEqual(os.getcwd(), root_dir) raise RuntimeError() + dirs = [] + def _chdir(path): + dirs.append(path) + orig_chdir(path) - register_archive_format('xxx', _breaks, [], 'xxx file') + register_archive_format('xxx', archiver, [], 'xxx file') try: - try: - make_archive('xxx', 'xxx', root_dir=self.mkdtemp()) - except Exception: - pass + with support.swap_attr(os, 'chdir', _chdir) as orig_chdir: + with self.assertRaises(RuntimeError): + make_archive('basename', 'xxx', root_dir=root_dir) + self.assertEqual(os.getcwd(), current_dir) + self.assertEqual(dirs, [root_dir, current_dir]) + finally: + unregister_archive_format('xxx') + + def test_make_archive_cwd_supports_root_dir(self): + current_dir = os.getcwd() + root_dir = self.mkdtemp() + def archiver(base_name, base_dir, **kw): + self.assertEqual(base_name, 'basename') + self.assertEqual(kw['root_dir'], root_dir) + self.assertEqual(os.getcwd(), current_dir) + raise RuntimeError() + archiver.supports_root_dir = True + + register_archive_format('xxx', archiver, [], 'xxx file') + try: + with no_chdir: + with self.assertRaises(RuntimeError): + make_archive('basename', 'xxx', root_dir=root_dir) self.assertEqual(os.getcwd(), current_dir) finally: unregister_archive_format('xxx') @@ -1431,15 +1885,15 @@ def _breaks(*args, **kw): def test_make_tarfile_in_curdir(self): # Issue #21280 root_dir = self.mkdtemp() - with os_helper.change_cwd(root_dir): + with os_helper.change_cwd(root_dir), no_chdir: self.assertEqual(make_archive('test', 'tar'), 'test.tar') self.assertTrue(os.path.isfile('test.tar')) - @support.requires_zlib + @support.requires_zlib() def test_make_zipfile_in_curdir(self): # Issue #21280 root_dir = self.mkdtemp() - with os_helper.change_cwd(root_dir): + with os_helper.change_cwd(root_dir), no_chdir: self.assertEqual(make_archive('test', 'zip'), 'test.zip') self.assertTrue(os.path.isfile('test.zip')) @@ -1459,12 +1913,59 @@ def test_register_archive_format(self): formats = [name for name, params in get_archive_formats()] self.assertNotIn('xxx', formats) - def check_unpack_archive(self, format): - self.check_unpack_archive_with_converter(format, lambda path: path) - self.check_unpack_archive_with_converter(format, pathlib.Path) - self.check_unpack_archive_with_converter(format, FakePath) - - def check_unpack_archive_with_converter(self, format, converter): + def test_make_tarfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'tar', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'tar', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + @support.requires_zlib() + def test_make_zipfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'zip', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'zip', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + + ### shutil.unpack_archive + + def check_unpack_archive(self, format, **kwargs): + self.check_unpack_archive_with_converter( + format, lambda path: path, **kwargs) + self.check_unpack_archive_with_converter( + format, pathlib.Path, **kwargs) + self.check_unpack_archive_with_converter(format, FakePath, **kwargs) + + def check_unpack_archive_with_converter(self, format, converter, **kwargs): root_dir, base_dir = self._create_files() expected = rlistdir(root_dir) expected.remove('outer') @@ -1474,36 +1975,52 @@ def check_unpack_archive_with_converter(self, format, converter): # let's try to unpack it now tmpdir2 = self.mkdtemp() - unpack_archive(converter(filename), converter(tmpdir2)) + unpack_archive(converter(filename), converter(tmpdir2), **kwargs) self.assertEqual(rlistdir(tmpdir2), expected) # and again, this time with the format specified tmpdir3 = self.mkdtemp() - unpack_archive(converter(filename), converter(tmpdir3), format=format) + unpack_archive(converter(filename), converter(tmpdir3), format=format, + **kwargs) self.assertEqual(rlistdir(tmpdir3), expected) - self.assertRaises(shutil.ReadError, unpack_archive, converter(TESTFN)) - self.assertRaises(ValueError, unpack_archive, converter(TESTFN), format='xxx') + with self.assertRaises(shutil.ReadError): + unpack_archive(converter(TESTFN), **kwargs) + with self.assertRaises(ValueError): + unpack_archive(converter(TESTFN), format='xxx', **kwargs) + + def check_unpack_tarball(self, format): + self.check_unpack_archive(format, filter='fully_trusted') + self.check_unpack_archive(format, filter='data') + with warnings_helper.check_warnings( + ('Python 3.14', DeprecationWarning)): + self.check_unpack_archive(format) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_unpack_archive_tar(self): - self.check_unpack_archive('tar') + self.check_unpack_tarball('tar') - @support.requires_zlib + # TODO: RUSTPYTHON + @unittest.expectedFailure + @support.requires_zlib() def test_unpack_archive_gztar(self): - self.check_unpack_archive('gztar') + self.check_unpack_tarball('gztar') - @support.requires_bz2 + @support.requires_bz2() def test_unpack_archive_bztar(self): - self.check_unpack_archive('bztar') + self.check_unpack_tarball('bztar') - @support.requires_lzma + @support.requires_lzma() @unittest.skipIf(AIX and not _maxdataOK(), "AIX MAXDATA must be 0x20000000 or larger") def test_unpack_archive_xztar(self): - self.check_unpack_archive('xztar') + self.check_unpack_tarball('xztar') - @support.requires_zlib + @support.requires_zlib() def test_unpack_archive_zip(self): self.check_unpack_archive('zip') + with self.assertRaises(TypeError): + self.check_unpack_archive('zip', filter='data') def test_unpack_registry(self): @@ -1531,6 +2048,9 @@ def _boo(filename, extract_dir, extra): unregister_unpack_format('Boo2') self.assertEqual(get_unpack_formats(), formats) + +class TestMisc(BaseTest, unittest.TestCase): + @unittest.skipUnless(hasattr(shutil, 'disk_usage'), "disk_usage not available on this platform") def test_disk_usage(self): @@ -1549,8 +2069,6 @@ def test_disk_usage(self): @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') def test_chown(self): - - # cleaned-up automatically by TestShutil.tearDown method dirname = self.mkdtemp() filename = tempfile.mktemp(dir=dirname) write_file(filename, 'testing chown function') @@ -1598,76 +2116,23 @@ def check_chown(path, uid=None, gid=None): shutil.chown(dirname, group=gid) check_chown(dirname, gid=gid) - user = pwd.getpwuid(uid)[0] - group = grp.getgrgid(gid)[0] - shutil.chown(filename, user, group) - check_chown(filename, uid, gid) - shutil.chown(dirname, user, group) - check_chown(dirname, uid, gid) - - def test_copy_return_value(self): - # copy and copy2 both return their destination path. - for fn in (shutil.copy, shutil.copy2): - src_dir = self.mkdtemp() - dst_dir = self.mkdtemp() - src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') - rv = fn(src, dst_dir) - self.assertEqual(rv, os.path.join(dst_dir, 'foo')) - rv = fn(src, os.path.join(dst_dir, 'bar')) - self.assertEqual(rv, os.path.join(dst_dir, 'bar')) - - def test_copyfile_return_value(self): - # copytree returns its destination path. - src_dir = self.mkdtemp() - dst_dir = self.mkdtemp() - dst_file = os.path.join(dst_dir, 'bar') - src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') - rv = shutil.copyfile(src_file, dst_file) - self.assertTrue(os.path.exists(rv)) - self.assertEqual(read_file(src_file), read_file(dst_file)) - - def test_copyfile_same_file(self): - # copyfile() should raise SameFileError if the source and destination - # are the same. - src_dir = self.mkdtemp() - src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') - self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file) - # But Error should work too, to stay backward compatible. - self.assertRaises(Error, shutil.copyfile, src_file, src_file) - # Make sure file is not corrupted. - self.assertEqual(read_file(src_file), 'foo') - - def test_copytree_return_value(self): - # copytree returns its destination path. - src_dir = self.mkdtemp() - dst_dir = src_dir + "dest" - self.addCleanup(shutil.rmtree, dst_dir, True) - src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') - rv = shutil.copytree(src_dir, dst_dir) - self.assertEqual(['foo'], os.listdir(rv)) - - def test_copytree_subdirectory(self): - # copytree where dst is a subdirectory of src, see Issue 38688 - base_dir = self.mkdtemp() - self.addCleanup(shutil.rmtree, base_dir, ignore_errors=True) - src_dir = os.path.join(base_dir, "t", "pg") - dst_dir = os.path.join(src_dir, "somevendor", "1.0") - os.makedirs(src_dir) - src = os.path.join(src_dir, 'pol') - write_file(src, 'pol') - rv = shutil.copytree(src_dir, dst_dir) - self.assertEqual(['pol'], os.listdir(rv)) + try: + user = pwd.getpwuid(uid)[0] + group = grp.getgrgid(gid)[0] + except KeyError: + # On some systems uid/gid cannot be resolved. + pass + else: + shutil.chown(filename, user, group) + check_chown(filename, uid, gid) + shutil.chown(dirname, user, group) + check_chown(dirname, uid, gid) -class TestWhich(unittest.TestCase): +class TestWhich(BaseTest, unittest.TestCase): def setUp(self): - self.temp_dir = tempfile.mkdtemp(prefix="Tmp") - self.addCleanup(shutil.rmtree, self.temp_dir, True) + self.temp_dir = self.mkdtemp(prefix="Tmp") # Give the temp_file an ".exe" suffix for all. # It's needed on Windows and not harmful on other platforms. self.temp_file = tempfile.NamedTemporaryFile(dir=self.temp_dir, @@ -1680,6 +2145,14 @@ def setUp(self): self.curdir = os.curdir self.ext = ".EXE" + def to_text_type(self, s): + ''' + In this class we're testing with str, so convert s to a str + ''' + if isinstance(s, bytes): + return s.decode() + return s + def test_basic(self): # Given an EXE in a directory, it should be returned. rv = shutil.which(self.file, path=self.dir) @@ -1704,20 +2177,69 @@ def test_relative_cmd(self): rv = shutil.which(relpath, path=base_dir) self.assertIsNone(rv) - def test_cwd(self): + @unittest.skipUnless(sys.platform != "win32", + "test is for non win32") + def test_cwd_non_win32(self): # Issue #16957 base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): rv = shutil.which(self.file, path=base_dir) - if sys.platform == "win32": - # Windows: current directory implicitly on PATH + # non-win32: shouldn't match in the current directory. + self.assertIsNone(rv) + + @unittest.skipUnless(sys.platform == "win32", + "test is for win32") + def test_cwd_win32(self): + base_dir = os.path.dirname(self.dir) + with os_helper.change_cwd(path=self.dir): + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + rv = shutil.which(self.file, path=base_dir) + # Current directory implicitly on PATH self.assertEqual(rv, os.path.join(self.curdir, self.file)) - else: - # Other platforms: shouldn't match in the current directory. + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=False): + rv = shutil.which(self.file, path=base_dir) + # Current directory not on PATH self.assertIsNone(rv) - @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, - 'non-root user required') + @unittest.skipUnless(sys.platform == "win32", + "test is for win32") + def test_cwd_win32_added_before_all_other_path(self): + base_dir = pathlib.Path(os.fsdecode(self.dir)) + + elsewhere_in_path_dir = base_dir / 'dir1' + elsewhere_in_path_dir.mkdir() + match_elsewhere_in_path = elsewhere_in_path_dir / 'hello.exe' + match_elsewhere_in_path.touch() + + exe_in_cwd = base_dir / 'hello.exe' + exe_in_cwd.touch() + + with os_helper.change_cwd(path=base_dir): + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + rv = shutil.which('hello.exe', path=elsewhere_in_path_dir) + + self.assertEqual(os.path.abspath(rv), os.path.abspath(exe_in_cwd)) + + @unittest.skipUnless(sys.platform == "win32", + "test is for win32") + def test_pathext_match_before_path_full_match(self): + base_dir = pathlib.Path(os.fsdecode(self.dir)) + dir1 = base_dir / 'dir1' + dir2 = base_dir / 'dir2' + dir1.mkdir() + dir2.mkdir() + + pathext_match = dir1 / 'hello.com.exe' + path_match = dir2 / 'hello.com' + pathext_match.touch() + path_match.touch() + + test_path = os.pathsep.join([str(dir1), str(dir2)]) + assert os.path.basename(shutil.which( + 'hello.com', path=test_path, mode = os.F_OK + )).lower() == 'hello.com.exe' + + @os_helper.skip_if_dac_override def test_non_matching_mode(self): # Set the file read-only and ask for writeable files. os.chmod(self.temp_file.name, stat.S_IREAD) @@ -1818,9 +2340,9 @@ def test_empty_path_no_PATH(self): @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext(self): - ext = ".xyz" + ext = self.to_text_type(".xyz") temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix="Tmp2", suffix=ext) + prefix=self.to_text_type("Tmp2"), suffix=ext) os.chmod(temp_filexyz.name, stat.S_IXUSR) self.addCleanup(temp_filexyz.close) @@ -1829,10 +2351,98 @@ def test_pathext(self): program = os.path.splitext(program)[0] with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = ext + env['PATHEXT'] = ext if isinstance(ext, str) else ext.decode() rv = shutil.which(program, path=self.temp_dir) self.assertEqual(rv, temp_filexyz.name) + # Issue 40592: See https://bugs.python.org/issue40592 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_empty_str(self): + ext = self.to_text_type(".xyz") + temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, + prefix=self.to_text_type("Tmp2"), suffix=ext) + self.addCleanup(temp_filexyz.close) + + # strip path and extension + program = os.path.basename(temp_filexyz.name) + program = os.path.splitext(program)[0] + + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = f"{ext if isinstance(ext, str) else ext.decode()};" # note the ; + rv = shutil.which(program, path=self.temp_dir) + self.assertEqual(rv, temp_filexyz.name) + + # See GH-75586 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_applied_on_files_in_path(self): + with os_helper.EnvironmentVarGuard() as env: + env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() + env["PATHEXT"] = ".test" + + test_path = os.path.join(self.temp_dir, self.to_text_type("test_program.test")) + open(test_path, 'w').close() + os.chmod(test_path, 0o755) + + self.assertEqual(shutil.which(self.to_text_type("test_program")), test_path) + + # See GH-75586 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_win_path_needs_curdir(self): + with unittest.mock.patch('_winapi.NeedCurrentDirectoryForExePath', return_value=True) as need_curdir_mock: + self.assertTrue(shutil._win_path_needs_curdir('dontcare', os.X_OK)) + need_curdir_mock.assert_called_once_with('dontcare') + need_curdir_mock.reset_mock() + self.assertTrue(shutil._win_path_needs_curdir('dontcare', 0)) + need_curdir_mock.assert_not_called() + + with unittest.mock.patch('_winapi.NeedCurrentDirectoryForExePath', return_value=False) as need_curdir_mock: + self.assertFalse(shutil._win_path_needs_curdir('dontcare', os.X_OK)) + need_curdir_mock.assert_called_once_with('dontcare') + + # See GH-109590 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_preferred_for_execute(self): + with os_helper.EnvironmentVarGuard() as env: + env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() + env["PATHEXT"] = ".test" + + exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) + open(exe, 'w').close() + os.chmod(exe, 0o755) + + # default behavior allows a direct match if nothing in PATHEXT matches + self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) + + dot_test = os.path.join(self.temp_dir, self.to_text_type("test.exe.test")) + open(dot_test, 'w').close() + os.chmod(dot_test, 0o755) + + # now we have a PATHEXT match, so it take precedence + self.assertEqual(shutil.which(self.to_text_type("test.exe")), dot_test) + + # but if we don't use os.X_OK we don't change the order based off PATHEXT + # and therefore get the direct match. + self.assertEqual(shutil.which(self.to_text_type("test.exe"), mode=os.F_OK), exe) + + # See GH-109590 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_given_extension_preferred(self): + with os_helper.EnvironmentVarGuard() as env: + env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() + env["PATHEXT"] = ".exe2;.exe" + + exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) + open(exe, 'w').close() + os.chmod(exe, 0o755) + + exe2 = os.path.join(self.temp_dir, self.to_text_type("test.exe2")) + open(exe2, 'w').close() + os.chmod(exe2, 0o755) + + # even though .exe2 is preferred in PATHEXT, we matched directly to test.exe + self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) + self.assertEqual(shutil.which(self.to_text_type("test")), exe2) + class TestWhichBytes(TestWhich): def setUp(self): @@ -1840,32 +2450,30 @@ def setUp(self): self.dir = os.fsencode(self.dir) self.file = os.fsencode(self.file) self.temp_file.name = os.fsencode(self.temp_file.name) + self.temp_dir = os.fsencode(self.temp_dir) self.curdir = os.fsencode(self.curdir) self.ext = os.fsencode(self.ext) + def to_text_type(self, s): + ''' + In this class we're testing with bytes, so convert s to a bytes + ''' + if isinstance(s, str): + return s.encode() + return s -class TestMove(unittest.TestCase): + +class TestMove(BaseTest, unittest.TestCase): def setUp(self): filename = "foo" - basedir = None - if sys.platform == "win32": - basedir = os.path.realpath(os.getcwd()) - self.src_dir = tempfile.mkdtemp(dir=basedir) - self.dst_dir = tempfile.mkdtemp(dir=basedir) + self.src_dir = self.mkdtemp() + self.dst_dir = self.mkdtemp() self.src_file = os.path.join(self.src_dir, filename) self.dst_file = os.path.join(self.dst_dir, filename) with open(self.src_file, "wb") as f: f.write(b"spam") - def tearDown(self): - for d in (self.src_dir, self.dst_dir): - try: - if d: - shutil.rmtree(d) - except: - pass - def _check_move_file(self, src, dst, real_dst): with open(src, "rb") as f: contents = f.read() @@ -1888,6 +2496,16 @@ def test_move_file_to_dir(self): # Move a file inside an existing dir on the same filesystem. self._check_move_file(self.src_file, self.dst_dir, self.dst_file) + def test_move_file_to_dir_pathlike_src(self): + # Move a pathlike file to another location on the same filesystem. + src = pathlib.Path(self.src_file) + self._check_move_file(src, self.dst_dir, self.dst_file) + + def test_move_file_to_dir_pathlike_dst(self): + # Move a file to another pathlike location on the same filesystem. + dst = pathlib.Path(self.dst_dir) + self._check_move_file(self.src_file, dst, self.dst_file) + @mock_rename def test_move_file_other_fs(self): # Move a file to an existing dir on another filesystem. @@ -1900,14 +2518,11 @@ def test_move_file_to_dir_other_fs(self): def test_move_dir(self): # Move a dir to another location on the same filesystem. - dst_dir = tempfile.mktemp() + dst_dir = tempfile.mktemp(dir=self.mkdtemp()) try: self._check_move_dir(self.src_dir, dst_dir, dst_dir) finally: - try: - shutil.rmtree(dst_dir) - except: - pass + os_helper.rmtree(dst_dir) @mock_rename def test_move_dir_other_fs(self): @@ -1954,7 +2569,7 @@ def test_destinsrc_false_negative(self): msg='_destinsrc() wrongly concluded that ' 'dst (%s) is not in src (%s)' % (dst, src)) finally: - shutil.rmtree(TESTFN, ignore_errors=True) + os_helper.rmtree(TESTFN) def test_destinsrc_false_positive(self): os.mkdir(TESTFN) @@ -1966,7 +2581,7 @@ def test_destinsrc_false_positive(self): msg='_destinsrc() wrongly concluded that ' 'dst (%s) is in src (%s)' % (dst, src)) finally: - shutil.rmtree(TESTFN, ignore_errors=True) + os_helper.rmtree(TESTFN) @os_helper.skip_unless_symlink @mock_rename @@ -2038,10 +2653,88 @@ def _copy(src, dst): shutil.move(self.src_dir, self.dst_dir, copy_function=_copy) self.assertEqual(len(moved), 3) + def test_move_dir_caseinsensitive(self): + # Renames a folder to the same name + # but a different case. -class TestCopyFile(unittest.TestCase): + self.src_dir = self.mkdtemp() + dst_dir = os.path.join( + os.path.dirname(self.src_dir), + os.path.basename(self.src_dir).upper()) + self.assertNotEqual(self.src_dir, dst_dir) + + try: + shutil.move(self.src_dir, dst_dir) + self.assertTrue(os.path.isdir(dst_dir)) + finally: + os.rmdir(dst_dir) + + # bpo-26791: Check that a symlink to a directory can + # be moved into that directory. + @mock_rename + def _test_move_symlink_to_dir_into_dir(self, dst): + src = os.path.join(self.src_dir, 'linktodir') + dst_link = os.path.join(self.dst_dir, 'linktodir') + os.symlink(self.dst_dir, src, target_is_directory=True) + shutil.move(src, dst) + self.assertTrue(os.path.islink(dst_link)) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertFalse(os.path.exists(src)) + + # Repeat the move operation with the destination + # symlink already in place (should raise shutil.Error). + os.symlink(self.dst_dir, src, target_is_directory=True) + with self.assertRaises(shutil.Error): + shutil.move(src, dst) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertTrue(os.path.exists(src)) - _delete = False + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_dir(self): + self._test_move_symlink_to_dir_into_dir(self.dst_dir) + + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_symlink_to_dir(self): + dst = os.path.join(self.src_dir, 'otherlinktodir') + os.symlink(self.dst_dir, dst, target_is_directory=True) + self._test_move_symlink_to_dir_into_dir(dst) + + @os_helper.skip_unless_dac_override + @unittest.skipUnless(hasattr(os, 'lchflags') + and hasattr(stat, 'SF_IMMUTABLE') + and hasattr(stat, 'UF_OPAQUE'), + 'requires lchflags') + def test_move_dir_permission_denied(self): + # bpo-42782: shutil.move should not create destination directories + # if the source directory cannot be removed. + try: + os.mkdir(TESTFN_SRC) + os.lchflags(TESTFN_SRC, stat.SF_IMMUTABLE) + + # Testing on an empty immutable directory + # TESTFN_DST should not exist if shutil.move failed + self.assertRaises(PermissionError, shutil.move, TESTFN_SRC, TESTFN_DST) + self.assertFalse(TESTFN_DST in os.listdir()) + + # Create a file and keep the directory immutable + os.lchflags(TESTFN_SRC, stat.UF_OPAQUE) + os_helper.create_empty_file(os.path.join(TESTFN_SRC, 'child')) + os.lchflags(TESTFN_SRC, stat.SF_IMMUTABLE) + + # Testing on a non-empty immutable directory + # TESTFN_DST should not exist if shutil.move failed + self.assertRaises(PermissionError, shutil.move, TESTFN_SRC, TESTFN_DST) + self.assertFalse(TESTFN_DST in os.listdir()) + finally: + if os.path.exists(TESTFN_SRC): + os.lchflags(TESTFN_SRC, stat.UF_OPAQUE) + os_helper.rmtree(TESTFN_SRC) + if os.path.exists(TESTFN_DST): + os.lchflags(TESTFN_DST, stat.UF_OPAQUE) + os_helper.rmtree(TESTFN_DST) + + +class TestCopyFile(unittest.TestCase): class Faux(object): _entered = False @@ -2061,27 +2754,18 @@ def __exit__(self, exc_type, exc_val, exc_tb): raise OSError("Cannot close") return self._suppress_at_exit - def tearDown(self): - if self._delete: - del shutil.open - - def _set_shutil_open(self, func): - shutil.open = func - self._delete = True - def test_w_source_open_fails(self): def _open(filename, mode='r'): if filename == 'srcfile': raise OSError('Cannot open "srcfile"') assert 0 # shouldn't reach here. - self._set_shutil_open(_open) - - self.assertRaises(OSError, shutil.copyfile, 'srcfile', 'destfile') + with support.swap_attr(shutil, 'open', _open): + with self.assertRaises(OSError): + shutil.copyfile('srcfile', 'destfile') @unittest.skipIf(MACOS, "skipped on macOS") def test_w_dest_open_fails(self): - srcfile = self.Faux() def _open(filename, mode='r'): @@ -2091,9 +2775,8 @@ def _open(filename, mode='r'): raise OSError('Cannot open "destfile"') assert 0 # shouldn't reach here. - self._set_shutil_open(_open) - - shutil.copyfile('srcfile', 'destfile') + with support.swap_attr(shutil, 'open', _open): + shutil.copyfile('srcfile', 'destfile') self.assertTrue(srcfile._entered) self.assertTrue(srcfile._exited_with[0] is OSError) self.assertEqual(srcfile._exited_with[1].args, @@ -2101,7 +2784,6 @@ def _open(filename, mode='r'): @unittest.skipIf(MACOS, "skipped on macOS") def test_w_dest_close_fails(self): - srcfile = self.Faux() destfile = self.Faux(True) @@ -2112,9 +2794,8 @@ def _open(filename, mode='r'): return destfile assert 0 # shouldn't reach here. - self._set_shutil_open(_open) - - shutil.copyfile('srcfile', 'destfile') + with support.swap_attr(shutil, 'open', _open): + shutil.copyfile('srcfile', 'destfile') self.assertTrue(srcfile._entered) self.assertTrue(destfile._entered) self.assertTrue(destfile._raised) @@ -2135,33 +2816,15 @@ def _open(filename, mode='r'): return destfile assert 0 # shouldn't reach here. - self._set_shutil_open(_open) - - self.assertRaises(OSError, - shutil.copyfile, 'srcfile', 'destfile') + with support.swap_attr(shutil, 'open', _open): + with self.assertRaises(OSError): + shutil.copyfile('srcfile', 'destfile') self.assertTrue(srcfile._entered) self.assertTrue(destfile._entered) self.assertFalse(destfile._raised) self.assertTrue(srcfile._exited_with[0] is None) self.assertTrue(srcfile._raised) - def test_move_dir_caseinsensitive(self): - # Renames a folder to the same name - # but a different case. - - self.src_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.src_dir, True) - dst_dir = os.path.join( - os.path.dirname(self.src_dir), - os.path.basename(self.src_dir).upper()) - self.assertNotEqual(self.src_dir, dst_dir) - - try: - shutil.move(self.src_dir, dst_dir) - self.assertTrue(os.path.isdir(dst_dir)) - finally: - os.rmdir(dst_dir) - class TestCopyFileObj(unittest.TestCase): FILESIZE = 2 * 1024 * 1024 @@ -2218,7 +2881,7 @@ def test_win_impl(self): # If file size < 1 MiB memoryview() length must be equal to # the actual file size. - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(dir=os.getcwd(), delete=False) as f: f.write(b'foo') fname = f.name self.addCleanup(os_helper.unlink, fname) @@ -2227,7 +2890,7 @@ def test_win_impl(self): self.assertEqual(m.call_args[0][2], 3) # Empty files should not rely on readinto() variant. - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(dir=os.getcwd(), delete=False) as f: pass fname = f.name self.addCleanup(os_helper.unlink, fname) @@ -2287,13 +2950,13 @@ def test_regular_copy(self): def test_same_file(self): self.addCleanup(self.reset) with self.get_files() as (src, dst): - with self.assertRaises(Exception): + with self.assertRaises((OSError, _GiveupOnFastCopy)): self.zerocopy_fun(src, src) # Make sure src file is not corrupted. self.assertEqual(read_file(TESTFN, binary=True), self.FILEDATA) def test_non_existent_src(self): - name = tempfile.mktemp() + name = tempfile.mktemp(dir=os.getcwd()) with self.assertRaises(FileNotFoundError) as cm: shutil.copyfile(name, "new") self.assertEqual(cm.exception.filename, name) @@ -2466,7 +3129,7 @@ def zerocopy_fun(self, src, dst): return shutil._fastcopy_fcopyfile(src, dst, posix._COPYFILE_DATA) -class TermsizeTests(unittest.TestCase): +class TestGetTerminalSize(unittest.TestCase): def test_does_not_crash(self): """Check if get_terminal_size() returns a meaningful value. @@ -2524,6 +3187,7 @@ def test_stty_match(self): self.assertEqual(expected, actual) + @unittest.skipIf(support.is_wasi, "WASI has no /dev/null") def test_fallback(self): with os_helper.EnvironmentVarGuard() as env: del env['LINES'] @@ -2537,7 +3201,7 @@ def test_fallback(self): # sys.__stdout__ is not a terminal on Unix # or fileno() not in (0, 1, 2) on Windows - with open(os.devnull, 'w') as f, \ + with open(os.devnull, 'w', encoding='utf-8') as f, \ support.swap_attr(sys, '__stdout__', f): size = shutil.get_terminal_size(fallback=(30, 40)) self.assertEqual(size.columns, 30) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index bd76c8ed95..f5c0ef923f 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -737,6 +737,21 @@ pub(super) mod _os { Ok(self.ino.load()) } + #[cfg(not(windows))] + #[pymethod] + fn is_junction(&self, _vm: &VirtualMachine) -> PyResult { + Ok(false) + } + + // TODO: RUSTPYTHON + // Check is_junction method logic is correct. + #[cfg(windows)] + #[pymethod] + fn is_junction(&self, _vm: &VirtualMachine) -> PyResult { + fs::metadata(&path).map_or(false, |meta| meta.file_type().is_dir()) + && fs::symlink_metadata(&path).map_or(false, |meta| meta.file_type().is_symlink()) + } + #[pymethod(magic)] fn fspath(&self, vm: &VirtualMachine) -> PyResult { self.path(vm) From 4d11adfc72710e87ecae7edc66a42140ae88b0bd Mon Sep 17 00:00:00 2001 From: Moreal Date: Tue, 16 Apr 2024 23:02:06 +0900 Subject: [PATCH 2/3] Implement windows platform target `is_junction` with junction library --- Cargo.lock | 11 +++++++++++ vm/Cargo.toml | 3 +++ vm/src/stdlib/os.rs | 5 +---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 01118ff8a9..ab03881fc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1055,6 +1055,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "junction" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca39ef0d69b18e6a2fd14c2f0a1d593200f4a4ed949b240b5917ab51fac754cb" +dependencies = [ + "scopeguard", + "winapi", +] + [[package]] name = "keccak" version = "0.1.3" @@ -2305,6 +2315,7 @@ dependencies = [ "indexmap 2.2.6", "is-macro", "itertools 0.11.0", + "junction", "libc", "log", "malachite-bigint", diff --git a/vm/Cargo.toml b/vm/Cargo.toml index f9061ba0b0..dc248a1667 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -106,6 +106,9 @@ schannel = { workspace = true } widestring = { workspace = true } winreg = "0.10.1" +[target.'cfg(windows)'.dependencies.junction] +version = "1.0.0" + [target.'cfg(windows)'.dependencies.windows] version = "0.52.0" features = [ diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f5c0ef923f..092c42f6b2 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -743,13 +743,10 @@ pub(super) mod _os { Ok(false) } - // TODO: RUSTPYTHON - // Check is_junction method logic is correct. #[cfg(windows)] #[pymethod] fn is_junction(&self, _vm: &VirtualMachine) -> PyResult { - fs::metadata(&path).map_or(false, |meta| meta.file_type().is_dir()) - && fs::symlink_metadata(&path).map_or(false, |meta| meta.file_type().is_symlink()) + Ok(junction::exists(self.pathval.clone()).unwrap_or(false)) } #[pymethod(magic)] From e9b41911ad683c1288be81078fdea15f295c5907 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 17 Apr 2024 00:28:35 +0900 Subject: [PATCH 3/3] Add _winapi.NeedCurrentDirectoryForExePath --- stdlib/Cargo.toml | 1 + vm/src/stdlib/winapi.rs | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 97a290711a..4e77b9f720 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -122,6 +122,7 @@ features = [ "Win32_NetworkManagement_IpHelper", "Win32_NetworkManagement_Ndis", "Win32_Security_Cryptography", + "Win32_System_Environment", ] [target.'cfg(target_os = "macos")'.dependencies] diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index ed88fabceb..8fce91be44 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -246,6 +246,17 @@ mod _winapi { )) } + #[pyfunction] + fn NeedCurrentDirectoryForExePath(exe_name: PyStrRef) -> bool { + let exe_name = exe_name.as_str().to_wides_with_nul(); + let return_value = unsafe { + windows_sys::Win32::System::Environment::NeedCurrentDirectoryForExePathW( + exe_name.as_ptr(), + ) + }; + return_value != 0 + } + fn getenvironment(env: ArgMapping, vm: &VirtualMachine) -> PyResult> { let keys = env.mapping().keys(vm)?; let values = env.mapping().values(vm)?;