From 5d24b0cf4752346b559a82b98f815acc1a6233cc Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 22 Oct 2024 09:38:39 -0700 Subject: [PATCH 01/12] test(ci): test on windows and osx --- .github/workflows/test.yml | 22 ++++++++++++++++++++-- pyproject.toml | 8 ++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d32f6f793c..9a222dae2d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,12 +18,30 @@ jobs: test: name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }} - runs-on: ubuntu-latest strategy: matrix: python-version: ['3.11', '3.12', '3.13'] - numpy-version: ['1.25', '1.26', '2.0'] + numpy-version: ['1.25', '2'] dependency-set: ["minimal", "optional"] + os: ["ubuntu-latest"] + include: + - python-version: '3.11' + numpy-version: '1.25' + dependency-set: 'optional' + os: 'macos-latest' + - python-version: '3.13' + numpy-version: '2' + dependency-set: 'optional' + os: 'macos-latest' + - python-version: '3.11' + numpy-version: '1.25' + dependency-set: 'optional' + os: 'windows-latest' + - python-version: '3.13' + numpy-version: '2' + dependency-set: 'optional' + os: 'windows-latest' + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index ef2ccd9469..ae6bc53c5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,17 +135,17 @@ features = ["test", "extra"] [[tool.hatch.envs.test.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "1.26", "2.0"] +numpy = ["1.25", "2"] version = ["minimal"] [[tool.hatch.envs.test.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "1.26", "2.0"] +numpy = ["1.25", "2"] features = ["optional"] [[tool.hatch.envs.test.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "1.26", "2.0"] +numpy = ["1.25", "2"] features = ["gpu"] [tool.hatch.envs.test.scripts] @@ -166,7 +166,7 @@ features = ["test", "extra", "gpu"] [[tool.hatch.envs.gputest.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "1.26", "2.0"] +numpy = ["1.25", "2"] version = ["minimal"] [tool.hatch.envs.gputest.scripts] From dc113f6fbdc77025949d09bfd36503730be2992b Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 22 Oct 2024 09:42:11 -0700 Subject: [PATCH 02/12] add os to name --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a222dae2d..c9ec621be8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ concurrency: jobs: test: - name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }} + name: os=${{ matrix.os }}, py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }} strategy: matrix: From 9aeeaa1e2ce9de6401059d0ef35e53f3bee77d9c Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 22 Oct 2024 09:45:13 -0700 Subject: [PATCH 03/12] bail on making np upper bound super flexible --- .github/workflows/test.yml | 6 +++--- pyproject.toml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9ec621be8..c388ba31de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: python-version: ['3.11', '3.12', '3.13'] - numpy-version: ['1.25', '2'] + numpy-version: ['1.25', '2.1'] dependency-set: ["minimal", "optional"] os: ["ubuntu-latest"] include: @@ -30,7 +30,7 @@ jobs: dependency-set: 'optional' os: 'macos-latest' - python-version: '3.13' - numpy-version: '2' + numpy-version: '2.1' dependency-set: 'optional' os: 'macos-latest' - python-version: '3.11' @@ -38,7 +38,7 @@ jobs: dependency-set: 'optional' os: 'windows-latest' - python-version: '3.13' - numpy-version: '2' + numpy-version: '2.1' dependency-set: 'optional' os: 'windows-latest' runs-on: ${{ matrix.os }} diff --git a/pyproject.toml b/pyproject.toml index ae6bc53c5e..39dd1155b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,17 +135,17 @@ features = ["test", "extra"] [[tool.hatch.envs.test.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2"] +numpy = ["1.25", "2.1"] version = ["minimal"] [[tool.hatch.envs.test.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2"] +numpy = ["1.25", "2.1"] features = ["optional"] [[tool.hatch.envs.test.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2"] +numpy = ["1.25", "2.1"] features = ["gpu"] [tool.hatch.envs.test.scripts] @@ -166,7 +166,7 @@ features = ["test", "extra", "gpu"] [[tool.hatch.envs.gputest.matrix]] python = ["3.11", "3.12", "3.13"] -numpy = ["1.25", "2"] +numpy = ["1.25", "2.1"] version = ["minimal"] [tool.hatch.envs.gputest.scripts] From 826cf0ac4e7806c7aa3d4f9b5b0c29043703b614 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 22 Oct 2024 09:53:30 -0700 Subject: [PATCH 04/12] remove lmdb --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 39dd1155b5..7be1999480 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,6 @@ test = [ "pytest", "pytest-cov", "msgpack", - "lmdb", "s3fs", "pytest-asyncio", "moto[s3]", @@ -92,13 +91,11 @@ docs = [ 'numpydoc', 'numcodecs[msgpack]', 'msgpack', - 'lmdb', ] extra = [ 'msgpack', ] optional = [ - 'lmdb', 'universal-pathlib>=0.0.22', ] From d8ae7d668955b35e01353d2e4645ee3eb8e3c100 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 22 Oct 2024 09:54:04 -0700 Subject: [PATCH 05/12] remove types-redis --- .pre-commit-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4667b20de1..bf8944d7ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,8 +37,6 @@ repos: - universal-pathlib # Tests - pytest - # Zarr v2 - - types-redis - repo: https://github.com/scientific-python/cookie rev: 2024.08.19 hooks: From 0e542f7a5341cb37eb76bce9fbf87178aaed0bae Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 22 Oct 2024 20:06:42 -0700 Subject: [PATCH 06/12] fixup --- .github/workflows/gpu_test.yml | 2 +- tests/test_sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index f9e36e9c2e..0403a9c06e 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -25,7 +25,7 @@ jobs: strategy: matrix: python-version: ['3.11'] - numpy-version: ['2.0'] + numpy-version: ['2.1'] dependency-set: ["minimal"] steps: diff --git a/tests/test_sync.py b/tests/test_sync.py index 20dbf33d95..5fae1cf4f5 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -61,7 +61,7 @@ def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: def test_sync_timeout() -> None: - duration = 0.002 + duration = 0.004 async def foo() -> None: await asyncio.sleep(duration) From df24e619ba119c39c9c7d1c4322db57b708454d6 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 22 Oct 2024 20:14:14 -0700 Subject: [PATCH 07/12] tweak timeout --- tests/test_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 5fae1cf4f5..a6216a4852 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -61,13 +61,13 @@ def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: def test_sync_timeout() -> None: - duration = 0.004 + duration = 0.02 async def foo() -> None: await asyncio.sleep(duration) with pytest.raises(asyncio.TimeoutError): - sync(foo(), timeout=duration / 2) + sync(foo(), timeout=duration / 10) def test_sync_raises_if_no_coroutine(sync_loop: asyncio.AbstractEventLoop | None) -> None: From cdadb53dbc2c6626ee7d557ebb9216a6ba65705c Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 22 Oct 2024 20:38:01 -0700 Subject: [PATCH 08/12] to_posix in LocalStore --- .github/workflows/hypothesis.yaml | 2 +- src/zarr/storage/local.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 85d48bddb1..1029063ef4 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -26,7 +26,7 @@ jobs: strategy: matrix: python-version: ['3.11'] - numpy-version: ['1.26'] + numpy-version: ['2.1'] dependency-set: ["optional"] steps: diff --git a/src/zarr/storage/local.py b/src/zarr/storage/local.py index 5c03009a97..e5751d6ee6 100644 --- a/src/zarr/storage/local.py +++ b/src/zarr/storage/local.py @@ -219,17 +219,17 @@ async def exists(self, key: str) -> bool: async def list(self) -> AsyncGenerator[str, None]: # docstring inherited - to_strip = str(self.root) + "/" + to_strip = self.root.as_posix() + "/" for p in list(self.root.rglob("*")): if p.is_file(): - yield str(p).replace(to_strip, "") + yield p.as_posix().replace(to_strip, "") async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited - to_strip = os.path.join(str(self.root / prefix)) + to_strip = (self.root / prefix).as_posix() + "/" # TODO: fixme in 2430 for p in (self.root / prefix).rglob("*"): if p.is_file(): - yield str(p.relative_to(to_strip)) + yield p.as_posix().replace(to_strip, "") async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited @@ -239,6 +239,6 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: try: key_iter = base.iterdir() for key in key_iter: - yield str(key).replace(to_strip, "") + yield key.as_posix().replace(to_strip, "") except (FileNotFoundError, NotADirectoryError): pass From f4f74fc318cc2800c754340dfd63c88ad441b04d Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Thu, 24 Oct 2024 15:55:55 -0700 Subject: [PATCH 09/12] no mo windows --- .github/workflows/test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c388ba31de..3c75743380 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,14 +33,14 @@ jobs: numpy-version: '2.1' dependency-set: 'optional' os: 'macos-latest' - - python-version: '3.11' - numpy-version: '1.25' - dependency-set: 'optional' - os: 'windows-latest' - - python-version: '3.13' - numpy-version: '2.1' - dependency-set: 'optional' - os: 'windows-latest' + # - python-version: '3.11' + # numpy-version: '1.25' + # dependency-set: 'optional' + # os: 'windows-latest' + # - python-version: '3.13' + # numpy-version: '2.1' + # dependency-set: 'optional' + # os: 'windows-latest' runs-on: ${{ matrix.os }} steps: From df7084203cdb913ac58c31019254bf52c7b6475c Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 24 Oct 2024 15:59:14 -0700 Subject: [PATCH 10/12] Update src/zarr/storage/local.py --- src/zarr/storage/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/storage/local.py b/src/zarr/storage/local.py index e5751d6ee6..4ac99ad92d 100644 --- a/src/zarr/storage/local.py +++ b/src/zarr/storage/local.py @@ -226,7 +226,7 @@ async def list(self) -> AsyncGenerator[str, None]: async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited - to_strip = (self.root / prefix).as_posix() + "/" # TODO: fixme in 2430 + to_strip = (self.root / prefix).as_posix() + "/" # TODO: fixme in https://github.com/zarr-developers/zarr-python/issues/2438 for p in (self.root / prefix).rglob("*"): if p.is_file(): yield p.as_posix().replace(to_strip, "") From 85f72bec7dfb7bf5c087fa541a17cbb5fe5a59d8 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 24 Oct 2024 16:00:09 -0700 Subject: [PATCH 11/12] Update .github/workflows/test.yml --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c75743380..2866ed4b85 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,6 +33,7 @@ jobs: numpy-version: '2.1' dependency-set: 'optional' os: 'macos-latest' + # https://github.com/zarr-developers/zarr-python/issues/2438 # - python-version: '3.11' # numpy-version: '1.25' # dependency-set: 'optional' From 30b1982f39ff594438b96297de3c0c0569435ee8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Oct 2024 23:04:37 +0000 Subject: [PATCH 12/12] style: pre-commit fixes --- src/zarr/storage/local.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/zarr/storage/local.py b/src/zarr/storage/local.py index 4ac99ad92d..ba13c2c661 100644 --- a/src/zarr/storage/local.py +++ b/src/zarr/storage/local.py @@ -226,7 +226,9 @@ async def list(self) -> AsyncGenerator[str, None]: async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited - to_strip = (self.root / prefix).as_posix() + "/" # TODO: fixme in https://github.com/zarr-developers/zarr-python/issues/2438 + to_strip = ( + (self.root / prefix).as_posix() + "/" + ) # TODO: fixme in https://github.com/zarr-developers/zarr-python/issues/2438 for p in (self.root / prefix).rglob("*"): if p.is_file(): yield p.as_posix().replace(to_strip, "")