diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1c6c67e5..d30c1d44 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, windows-latest, macos-latest] - pyv: ['3.8', '3.9', '3.10', '3.11', '3.12'] + pyv: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] fsspec: [''] include: @@ -41,6 +41,7 @@ jobs: PIP_DISABLE_PIP_VERSION_CHECK: ${{ matrix.pyv == '3.8' && matrix.os == 'macos-latest' }} with: python-version: ${{ matrix.pyv }} + allow-prereleases: true - name: Upgrade pip and nox run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a309551..61280762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ... +## [0.2.5] - 2024-09-08 +### Fixed +- upath.implementations.cloud: move bucket check to subclasses (#277) +- upath: enable local tests on windows and fix is_absolute (#278) +- upath: updated flavour sources (#273) + +### Added +- upath: adds support for python-3.13 (#275) + ## [0.2.4] - 2024-09-07 ### Fixed - upath: fix UPath.rename type signature (#258) @@ -160,7 +169,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - started a changelog to keep track of significant changes -[Unreleased]: https://github.com/fsspec/universal_pathlib/compare/v0.2.4...HEAD +[Unreleased]: https://github.com/fsspec/universal_pathlib/compare/v0.2.5...HEAD +[0.2.5]: https://github.com/fsspec/universal_pathlib/compare/v0.2.4...v0.2.5 [0.2.4]: https://github.com/fsspec/universal_pathlib/compare/v0.2.3...v0.2.4 [0.2.3]: https://github.com/fsspec/universal_pathlib/compare/v0.2.2...v0.2.3 [0.2.2]: https://github.com/fsspec/universal_pathlib/compare/v0.2.1...v0.2.2 diff --git a/README.md b/README.md index 055bdb79..5acb4ff3 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ project as a dependency if you want to use it with `s3` and `http` filesystems: name = "myproject" requires-python = ">=3.8" dependencies = [ - "universal_pathlib>=0.2.0", + "universal_pathlib>=0.2.5", "fsspec[s3,http]", ] ``` @@ -83,7 +83,7 @@ For more examples, see the [example notebook here][example-notebook]. ### Currently supported filesystems (and protocols) -- `file:` Local filesystem +- `file:` and `local:` Local filesystem - `memory:` Ephemeral filesystem in RAM - `az:`, `adl:`, `abfs:` and `abfss:` Azure Storage _(requires `adlfs`)_ - `data:` RFC 2397 style data URLs _(requires `fsspec>=2023.12.2`)_ @@ -92,6 +92,8 @@ For more examples, see the [example notebook here][example-notebook]. - `hdfs:` Hadoop distributed filesystem - `gs:` and `gcs:` Google Cloud Storage _(requires `gcsfs`)_ - `s3:` and `s3a:` AWS S3 _(requires `s3fs` to be installed)_ +- `sftp:` and `ssh:` SFTP and SSH filesystems _(requires `paramiko`)_ +- `smb:` SMB filesystems _(requires `smbprotocol`)_ - `webdav`, `webdav+http:` and `webdav+https:` WebDAV-based filesystem on top of HTTP(S) _(requires `webdav4[fsspec]`)_ diff --git a/dev/fsspec_inspector/generate_flavours.py b/dev/fsspec_inspector/generate_flavours.py index d1e6fbc1..2bd95fb7 100644 --- a/dev/fsspec_inspector/generate_flavours.py +++ b/dev/fsspec_inspector/generate_flavours.py @@ -336,6 +336,8 @@ def generate_class_source_code( s.append(f" {attr} = {value!r}") for attr in attributes: s.append(f" {attr} = {getattr(cls, attr)!r}") + if getattr(cls, "local_file", False): + s.append(" local_file = True") s.append("") for method in methods: s.append(inspect.getsource(getattr(cls, method))) diff --git a/dev/requirements.txt b/dev/requirements.txt index c688d7ca..db3ffb84 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -10,7 +10,7 @@ ocifs==1.3.1 webdav4[fsspec]==0.10.0 # gfrivefs @ git+https://github.com/fsspec/gdrivefs@master broken ... morefs[asynclocalfs]==0.2.2 -dvc==3.55.1 +dvc==3.55.2 huggingface_hub==0.24.6 lakefs-spec==0.10.0 ossfs==2023.12.0 diff --git a/noxfile.py b/noxfile.py index 485c9d32..63d3ed2c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -10,7 +10,7 @@ locations = ("upath",) -@nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.8", "pypy3.9"]) +@nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]) def tests(session: nox.Session) -> None: # workaround in case no aiohttp binary wheels are available session.env["AIOHTTP_NO_EXTENSIONS"] = "1" diff --git a/pyproject.toml b/pyproject.toml index d6ee777f..54b638e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Development Status :: 4 - Beta", ] keywords = ["filesystem-spec", "pathlib"] @@ -44,12 +45,13 @@ tests = [ "packaging", ] dev = [ - "adlfs", + "adlfs; python_version<='3.12' or (python_version>'3.12' and os_name!='nt') ", "aiohttp", "requests", "gcsfs", "s3fs", - "moto[s3,server]", + # exclude moto installation on 3.13 windows until pywin32 wheels are available: + "moto[s3,server]; python_version<='3.12' or (python_version>'3.12' and os_name!='nt') ", "webdav4[fsspec]", "paramiko", "wsgidav", diff --git a/upath/_flavour_sources.py b/upath/_flavour_sources.py index bdeae3e7..42f10088 100644 --- a/upath/_flavour_sources.py +++ b/upath/_flavour_sources.py @@ -140,6 +140,7 @@ class AsyncLocalFileSystemFlavour(AbstractFileSystemFlavour): protocol = () root_marker = '/' sep = '/' + local_file = True @classmethod def _strip_protocol(cls, path): @@ -622,6 +623,7 @@ class LocalFileSystemFlavour(AbstractFileSystemFlavour): protocol = ('file', 'local') root_marker = '/' sep = '/' + local_file = True @classmethod def _strip_protocol(cls, path): @@ -958,7 +960,7 @@ def _strip_protocol(cls, path): class _DVCFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'dvc.fs.dvc._DVCFileSystem' - __orig_version__ = '3.55.1' + __orig_version__ = '3.55.2' protocol = ('dvc',) root_marker = '/' sep = '/' diff --git a/upath/core.py b/upath/core.py index 45efd609..2003fa95 100644 --- a/upath/core.py +++ b/upath/core.py @@ -135,6 +135,9 @@ def with_suffix(self, suffix: str) -> Self: ... _protocol_dispatch: bool | None = None _flavour = LazyFlavourDescriptor() + if sys.version_info >= (3, 13): + parser = _flavour + # === upath.UPath constructor ===================================== def __new__( @@ -868,7 +871,7 @@ def iterdir(self) -> Generator[UPath, None, None]: continue # only want the path name with iterdir _, _, name = str_remove_suffix(name, "/").rpartition(self._flavour.sep) - yield self._make_child_relpath(name) + yield self.with_segments(*self.parts, name) def _scandir(self): raise NotImplementedError # todo diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index 455fca6b..36f4029f 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -22,13 +22,6 @@ class CloudPath(UPath): __slots__ = () - def __init__( - self, *args, protocol: str | None = None, **storage_options: Any - ) -> None: - super().__init__(*args, protocol=protocol, **storage_options) - if not self.drive and len(self.parts) > 1: - raise ValueError("non key-like path provided (bucket/container missing)") - @classmethod def _transform_init_args( cls, @@ -71,6 +64,13 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): class GCSPath(CloudPath): __slots__ = () + def __init__( + self, *args, protocol: str | None = None, **storage_options: Any + ) -> None: + super().__init__(*args, protocol=protocol, **storage_options) + if not self.drive and len(self.parts) > 1: + raise ValueError("non key-like path provided (bucket/container missing)") + def mkdir( self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False ) -> None: @@ -84,6 +84,20 @@ def mkdir( class S3Path(CloudPath): __slots__ = () + def __init__( + self, *args, protocol: str | None = None, **storage_options: Any + ) -> None: + super().__init__(*args, protocol=protocol, **storage_options) + if not self.drive and len(self.parts) > 1: + raise ValueError("non key-like path provided (bucket/container missing)") + class AzurePath(CloudPath): __slots__ = () + + def __init__( + self, *args, protocol: str | None = None, **storage_options: Any + ) -> None: + super().__init__(*args, protocol=protocol, **storage_options) + if not self.drive and len(self.parts) > 1: + raise ValueError("non key-like path provided (bucket/container missing)") diff --git a/upath/tests/cases.py b/upath/tests/cases.py index f539c6d3..6c9b914d 100644 --- a/upath/tests/cases.py +++ b/upath/tests/cases.py @@ -532,6 +532,10 @@ def test_read_with_fsspec(self): with fs.open(path) as f: assert f.read() == b"hello world" + @pytest.mark.xfail( + sys.version_info >= (3, 13), + reason="no support for private `._drv`, `._root`, `._parts` in 3.13", + ) def test_access_to_private_api(self): # DO NOT access these private attributes in your code p = UPath(str(self.path), **self.path.storage_options) diff --git a/upath/tests/implementations/test_local.py b/upath/tests/implementations/test_local.py index 437c6f55..e3f59d48 100644 --- a/upath/tests/implementations/test_local.py +++ b/upath/tests/implementations/test_local.py @@ -3,11 +3,9 @@ from upath import UPath from upath.implementations.local import LocalPath from upath.tests.cases import BaseTests -from upath.tests.utils import skip_on_windows from upath.tests.utils import xfail_if_version -@skip_on_windows class TestFSSpecLocal(BaseTests): @pytest.fixture(autouse=True) def path(self, local_testdir): @@ -18,7 +16,6 @@ def test_is_LocalPath(self): assert isinstance(self.path, LocalPath) -@skip_on_windows @xfail_if_version("fsspec", lt="2023.10.0", reason="requires fsspec>=2023.10.0") class TestRayIOFSSpecLocal(BaseTests): @pytest.fixture(autouse=True)