diff --git a/.github/workflows/imports.yml b/.github/workflows/imports.yml index a9863a213..2b0b0ed9f 100644 --- a/.github/workflows/imports.yml +++ b/.github/workflows/imports.yml @@ -7,13 +7,24 @@ on: - main jobs: - test_imports: + rngs: runs-on: ubuntu-latest - # strategy: - # matrix: - # python-version: ["3.8", "3.9", "3.10"] + outputs: + os: ${{ steps.os.outputs.selected }} + pyver: ${{ steps.pyver.outputs.selected }} steps: - - uses: actions/checkout@v3 + - name: RNG for os + uses: ddradar/choose-random-action@v2.0.2 + id: os + with: + contents: | + ubuntu-latest + macos-latest + windows-latest + weights: | + 1 + 1 + 1 - name: RNG for Python version uses: ddradar/choose-random-action@v2.0.2 id: pyver @@ -22,14 +33,26 @@ jobs: 3.8 3.9 3.10 + 3.11 weights: | 1 1 1 + 1 + test_imports: + needs: rngs + runs-on: ${{ needs.rngs.outputs.os }} + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # python-version: ["3.8", "3.9", "3.10", "3.11"] + # os: ["ubuntu-latest", "macos-latest", "windows-latest"] + steps: + - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: ${{ steps.pyver.outputs.selected }} + python-version: ${{ needs.rngs.outputs.pyver }} # python-version: ${{ matrix.python-version }} - run: python -m pip install --upgrade pip - - run: pip install -e . + - run: pip install -e .[default] - run: ./scripts/test_imports.sh diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 04ffd3eb5..6aa692155 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -102,17 +102,19 @@ jobs: id: pyver with: # We should support major Python versions for at least 36-42 months - # We could probably support pypy if numba were optional + # We may be able to support pypy if anybody asks for it # 3.8.16 0_73_pypy # 3.9.16 0_73_pypy contents: | 3.8 3.9 3.10 + 3.11 weights: | 1 1 1 + 1 - name: RNG for source of python-suitesparse-graphblas uses: ddradar/choose-random-action@v2.0.2 id: sourcetype @@ -138,8 +140,8 @@ jobs: miniforge-version: latest use-mamba: true python-version: ${{ steps.pyver.outputs.selected }} - channels: conda-forge,nodefaults - channel-priority: strict + channels: conda-forge,${{ contains(steps.pyver.outputs.selected, 'pypy') && 'defaults' || 'nodefaults' }} + channel-priority: ${{ contains(steps.pyver.outputs.selected, 'pypy') && 'flexible' || 'strict' }} activate-environment: graphblas auto-activate-base: false - name: Setup conda @@ -150,8 +152,8 @@ jobs: with: auto-update-conda: true python-version: ${{ steps.pyver.outputs.selected }} - channels: conda-forge,nodefaults - channel-priority: strict + channels: conda-forge,${{ contains(steps.pyver.outputs.selected, 'pypy') && 'defaults' || 'nodefaults' }} + channel-priority: ${{ contains(steps.pyver.outputs.selected, 'pypy') && 'flexible' || 'strict' }} activate-environment: graphblas auto-activate-base: false - name: Update env @@ -161,29 +163,29 @@ jobs: # # First let's randomly get versions of dependencies to install. # Consider removing old versions when they become problematic or very old (>=2 years). - nxver=$(python -c 'import random ; print(random.choice(["=2.7", "=2.8", "=3.0", ""]))') + nxver=$(python -c 'import random ; print(random.choice(["=2.7", "=2.8", "=3.0", "=3.1", ""]))') yamlver=$(python -c 'import random ; print(random.choice(["=5.4", "=6.0", ""]))') - sparsever=$(python -c 'import random ; print(random.choice(["=0.12", "=0.13", "=0.14", ""]))') - fmmver=$(python -c 'import random ; print(random.choice(["=1.4", ""]))') + sparsever=$(python -c 'import random ; print(random.choice(["=0.13", "=0.14", ""]))') + fmmver=$(python -c 'import random ; print(random.choice(["=1.4", "=1.5", ""]))') if [[ ${{ startsWith(steps.pyver.outputs.selected, '3.8') }} == true ]]; then - npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", ""]))') + npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", "=1.24", ""]))') spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", ""]))') - pdver=$(python -c 'import random ; print(random.choice(["=1.2", "=1.3", "=1.4", "=1.5", ""]))') + pdver=$(python -c 'import random ; print(random.choice(["=1.2", "=1.3", "=1.4", "=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=2.0", "=2.1", ""]))') elif [[ ${{ startsWith(steps.pyver.outputs.selected, '3.9') }} == true ]]; then - npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", ""]))') + npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", "=1.24", ""]))') spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", ""]))') - pdver=$(python -c 'import random ; print(random.choice(["=1.2", "=1.3", "=1.4", "=1.5", ""]))') + pdver=$(python -c 'import random ; print(random.choice(["=1.2", "=1.3", "=1.4", "=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=2.0", "=2.1", ""]))') elif [[ ${{ startsWith(steps.pyver.outputs.selected, '3.10') }} == true ]]; then - npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", ""]))') + npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", "=1.24", ""]))') spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", ""]))') - pdver=$(python -c 'import random ; print(random.choice(["=1.3", "=1.4", "=1.5", ""]))') + pdver=$(python -c 'import random ; print(random.choice(["=1.3", "=1.4", "=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=2.0", "=2.1", ""]))') else # Python 3.11 - npver=$(python -c 'import random ; print(random.choice(["=1.23", ""]))') + npver=$(python -c 'import random ; print(random.choice(["=1.23", "=1.24", ""]))') spver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", ""]))') - pdver=$(python -c 'import random ; print(random.choice(["=1.5", ""]))') + pdver=$(python -c 'import random ; print(random.choice(["=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.10", "=2.0", "=2.1", ""]))') fi if [[ ${{ steps.sourcetype.outputs.selected }} == "source" || ${{ steps.sourcetype.outputs.selected }} == "upstream" ]]; then @@ -208,16 +210,54 @@ jobs: else psgver="" fi - if [[ $npver == "=1.21" ]] ; then - numbaver=$(python -c 'import random ; print(random.choice(["=0.55", "=0.56", ""]))') + # TODO: drop 0.57.0rc1 and use 0.57 once numba 0.57 is properly released + if [[ ${npver} == "=1.24" || ${{ startsWith(steps.pyver.outputs.selected, '3.11') }} == true ]] ; then + numbaver=$(python -c 'import random ; print(random.choice(["=0.57.0rc1", ""]))') + elif [[ ${npver} == "=1.21" ]] ; then + numbaver=$(python -c 'import random ; print(random.choice(["=0.55", "=0.56", "=0.57.0rc1", ""]))') + else + numbaver=$(python -c 'import random ; print(random.choice(["=0.56", "=0.57.0rc1", ""]))') + fi + if [[ ${{ matrix.os == 'windows-latest' }} == true && ( ${npver} == "=1.24" || ${numbaver} == "=0.57.0rc1" ) ]] ; then + # TODO: numba 0.57.0rc1 currently crashes sometimes on windows, so skip it for now + npver="" + numbaver="" + fi + fmm=fast_matrix_market${fmmver} + awkward=awkward${akver} + if [[ ${{ contains(steps.pyver.outputs.selected, 'pypy') || + startsWith(steps.pyver.outputs.selected, '3.12') }} == true || + ( ${{ matrix.slowtask != 'notebooks'}} == true && ( + ( ${{ matrix.os == 'windows-latest' }} == true && $(python -c 'import random ; print(random.random() < .2)') == True ) || + ( ${{ matrix.os == 'windows-latest' }} == false && $(python -c 'import random ; print(random.random() < .4)') == True ))) ]] + then + # Some packages aren't available for pypy or Python 3.12; randomly otherwise (if not running notebooks) + echo "skipping numba" + numba="" + numbaver=NA + sparse="" + sparsever=NA + if [[ ${{ contains(steps.pyver.outputs.selected, 'pypy') }} ]]; then + awkward="" + akver=NA + fmm="" + fmmver=NA + # Be more flexible until we determine what versions are supported by pypy + npver="" + spver="" + pdver="" + yamlver="" + fi else - numbaver=$(python -c 'import random ; print(random.choice(["=0.56", ""]))') + numba=numba${numbaver} + sparse=sparse${sparsever} fi echo "versions: np${npver} sp${spver} pd${pdver} ak${akver} nx${nxver} numba${numbaver} yaml${yamlver} sparse${sparsever} psgver${psgver}" - $(command -v mamba || command -v conda) install packaging pytest coverage coveralls=3.3.1 pytest-randomly cffi donfig tomli \ - pyyaml${yamlver} sparse${sparsever} pandas${pdver} scipy${spver} numpy${npver} awkward${akver} \ - networkx${nxver} numba${numbaver} fast_matrix_market${fmmver} ${psg} \ + # TODO: remove `-c numba` when numba 0.57 is properly released + $(command -v mamba || command -v conda) install -c numba packaging pytest coverage coveralls=3.3.1 pytest-randomly cffi donfig tomli \ + pyyaml${yamlver} ${sparse} pandas${pdver} scipy${spver} numpy${npver} ${awkward} \ + networkx${nxver} ${numba} ${fmm} ${psg} \ ${{ matrix.slowtask == 'pytest_bizarro' && 'black' || '' }} \ ${{ matrix.slowtask == 'notebooks' && 'matplotlib nbconvert jupyter "ipython>=7"' || '' }} \ ${{ steps.sourcetype.outputs.selected == 'upstream' && 'cython' || '' }} \ @@ -269,9 +309,9 @@ jobs: if [[ $G && $bizarro ]] ; then if [[ $ubuntu ]] ; then echo " $suitesparse" ; elif [[ $windows ]] ; then echo " $vanilla" ; fi ; fi)$( \ if [[ $H && $normal ]] ; then if [[ $macos ]] ; then echo " $vanilla" ; elif [[ $windows ]] ; then echo " $suitesparse" ; fi ; fi)$( \ if [[ $H && $bizarro ]] ; then if [[ $macos ]] ; then echo " $suitesparse" ; elif [[ $windows ]] ; then echo " $vanilla" ; fi ; fi) - echo $args + echo ${args} pytest -v --pyargs suitesparse_graphblas - coverage run -m pytest --color=yes --randomly -v $args \ + coverage run -m pytest --color=yes --randomly -v ${args} \ ${{ matrix.slowtask == 'pytest_normal' && '--runslow' || '' }} - name: Unit tests (bizarro scalars) run: | @@ -305,8 +345,8 @@ jobs: if [[ $G && $bizarro ]] ; then if [[ $ubuntu ]] ; then echo " $vanilla" ; elif [[ $windows ]] ; then echo " $suitesparse" ; fi ; fi)$( \ if [[ $H && $normal ]] ; then if [[ $macos ]] ; then echo " $suitesparse" ; elif [[ $windows ]] ; then echo " $vanilla" ; fi ; fi)$( \ if [[ $H && $bizarro ]] ; then if [[ $macos ]] ; then echo " $vanilla" ; elif [[ $windows ]] ; then echo " $suitesparse" ; fi ; fi) - echo $args - coverage run -a -m pytest --color=yes --randomly -v $args \ + echo ${args} + coverage run -a -m pytest --color=yes --randomly -v ${args} \ ${{ matrix.slowtask == 'pytest_bizarro' && '--runslow' || '' }} git checkout . # Undo changes to scalar default - name: Miscellaneous tests @@ -329,6 +369,13 @@ jobs: # TODO: understand why these are order-dependent and try to fix coverage run -a -m pytest --color=yes -x --no-mapnumpy --runslow -k test_binaryop_attributes_numpy graphblas/tests/test_op.py # coverage run -a -m pytest --color=yes -x --no-mapnumpy -k test_npmonoid graphblas/tests/test_numpyops.py --runslow + - name: More tests for coverage + if: matrix.slowtask == 'notebooks' && matrix.os == 'windows-latest' + run: | + # We use 'notebooks' slow task b/c it should have numba installed + coverage run -a -m pytest --color=yes --runslow --no-mapnumpy -p no:randomly -v -k 'test_commutes or test_bool_doesnt_get_too_large or test_npbinary or test_npmonoid or test_npsemiring' + coverage run -a -m pytest --color=yes --runslow --mapnumpy -p no:randomly -k 'test_bool_doesnt_get_too_large or test_npunary or test_binaryop_monoid_numpy' + coverage run -a -m pytest --color=yes -x --no-mapnumpy --runslow -k test_binaryop_attributes_numpy graphblas/tests/test_op.py - name: Auto-generated code check if: matrix.slowtask == 'pytest_bizarro' run: | @@ -364,7 +411,11 @@ jobs: uses: codecov/codecov-action@v3 - name: Notebooks Execution check if: matrix.slowtask == 'notebooks' - run: jupyter nbconvert --to notebook --execute notebooks/*ipynb + run: | + # Run notebooks only if numba is installed + if python -c 'import numba' 2> /dev/null ; then + jupyter nbconvert --to notebook --execute notebooks/*ipynb + fi finish: needs: build_and_test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b00aee30..426153fee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,7 +58,7 @@ repos: - id: black - id: black-jupyter - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.260 + rev: v0.0.261 hooks: - id: ruff args: [--fix-only, --show-fixes] @@ -86,7 +86,7 @@ repos: additional_dependencies: [tomli] files: ^(graphblas|docs)/ - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.260 + rev: v0.0.261 hooks: - id: ruff - repo: https://github.com/sphinx-contrib/sphinx-lint diff --git a/README.md b/README.md index 23fc3650d..f07fdea12 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ $ conda install -c conda-forge python-graphblas ``` or pip: ``` -$ pip install python-graphblas +$ pip install python-graphblas[default] ``` This will also install the [SuiteSparse:GraphBLAS](https://github.com/DrTimothyAldenDavis/GraphBLAS) compiled C library. diff --git a/docs/_static/img/GraphBLAS-API-example.png b/docs/_static/img/GraphBLAS-API-example.png index c6dd48182..1edc91988 100644 Binary files a/docs/_static/img/GraphBLAS-API-example.png and b/docs/_static/img/GraphBLAS-API-example.png differ diff --git a/docs/_static/img/GraphBLAS-mapping.png b/docs/_static/img/GraphBLAS-mapping.png index 7ef73c88d..c5d1a1d4e 100644 Binary files a/docs/_static/img/GraphBLAS-mapping.png and b/docs/_static/img/GraphBLAS-mapping.png differ diff --git a/docs/_static/img/Matrix-A-strictly-upper.png b/docs/_static/img/Matrix-A-strictly-upper.png index 9b127aa84..0fedf2617 100644 Binary files a/docs/_static/img/Matrix-A-strictly-upper.png and b/docs/_static/img/Matrix-A-strictly-upper.png differ diff --git a/docs/_static/img/Matrix-A-upper.png b/docs/_static/img/Matrix-A-upper.png index 1b930a9a3..e3703710a 100644 Binary files a/docs/_static/img/Matrix-A-upper.png and b/docs/_static/img/Matrix-A-upper.png differ diff --git a/docs/_static/img/Recorder-output.png b/docs/_static/img/Recorder-output.png index 355cc1376..525221c55 100644 Binary files a/docs/_static/img/Recorder-output.png and b/docs/_static/img/Recorder-output.png differ diff --git a/docs/_static/img/adj-graph.png b/docs/_static/img/adj-graph.png index da9f36447..13a05fcc2 100644 Binary files a/docs/_static/img/adj-graph.png and b/docs/_static/img/adj-graph.png differ diff --git a/docs/_static/img/draw-example.png b/docs/_static/img/draw-example.png index 3c5e6c008..90c5917d9 100644 Binary files a/docs/_static/img/draw-example.png and b/docs/_static/img/draw-example.png differ diff --git a/docs/_static/img/logo-name-dark.svg b/docs/_static/img/logo-name-dark.svg index 039eb7e25..cdf5227c7 100644 --- a/docs/_static/img/logo-name-dark.svg +++ b/docs/_static/img/logo-name-dark.svg @@ -1,6 +1,4 @@ - - - - - - - - - - - - __EXPR__", topline) + # BRANCH NOT COVERED keys = [] values = [] diff --git a/graphblas/core/matrix.py b/graphblas/core/matrix.py index 1935fcee7..0183893fd 100644 --- a/graphblas/core/matrix.py +++ b/graphblas/core/matrix.py @@ -7,7 +7,7 @@ from .. import backend, binary, monoid, select, semiring from ..dtypes import _INDEX, FP64, INT64, lookup_dtype, unify from ..exceptions import DimensionMismatch, InvalidValue, NoValue, check_status -from . import automethods, ffi, lib, utils +from . import _supports_udfs, automethods, ffi, lib, utils from .base import BaseExpression, BaseType, _check_mask, call from .descriptor import lookup as descriptor_lookup from .expr import _ALL_INDICES, AmbiguousAssignOrExtract, IndexerResolver, Updater @@ -33,7 +33,7 @@ values_to_numpy_buffer, wrapdoc, ) -from .vector import Vector, VectorExpression, VectorIndexExpr, _select_mask +from .vector import Vector, VectorExpression, VectorIndexExpr, _isclose_recipe, _select_mask if backend == "suitesparse": from .ss.matrix import ss @@ -368,6 +368,8 @@ def isclose(self, other, *, rel_tol=1e-7, abs_tol=0.0, check_dtype=False, **opts return False if self._nvals != other._nvals: return False + if not _supports_udfs: + return _isclose_recipe(self, other, rel_tol, abs_tol, **opts) matches = self.ewise_mult(other, binary.isclose(rel_tol, abs_tol)).new( bool, name="M_isclose", **opts @@ -611,14 +613,15 @@ def build(self, rows, columns, values, *, dup_op=None, clear=False, nrows=None, if not dup_op_given: if not self.dtype._is_udt: dup_op = binary.plus - else: + elif backend != "suitesparse": dup_op = binary.any - # SS:SuiteSparse-specific: we could use NULL for dup_op - dup_op = get_typed_op(dup_op, self.dtype, kind="binary") - if dup_op.opclass == "Monoid": - dup_op = dup_op.binaryop - else: - self._expect_op(dup_op, "BinaryOp", within="build", argname="dup_op") + # SS:SuiteSparse-specific: we use NULL for dup_op + if dup_op is not None: + dup_op = get_typed_op(dup_op, self.dtype, kind="binary") + if dup_op.opclass == "Monoid": + dup_op = dup_op.binaryop + else: + self._expect_op(dup_op, "BinaryOp", within="build", argname="dup_op") rows = _CArray(rows) columns = _CArray(columns) @@ -1584,7 +1587,7 @@ def from_dicts( # If we know the dtype, then using `np.fromiter` is much faster dtype = lookup_dtype(dtype) if dtype.np_type.subdtype is not None and np.__version__[:5] in {"1.21.", "1.22."}: - values, dtype = values_to_numpy_buffer(list(iter_values), dtype) + values, dtype = values_to_numpy_buffer(list(iter_values), dtype) # FLAKY COVERAGE else: values = np.fromiter(iter_values, dtype.np_type) return getattr(cls, methodname)( @@ -2466,6 +2469,7 @@ def select(self, op, thunk=None): self._expect_op(op, ("SelectOp", "IndexUnaryOp"), within=method_name, argname="op") if thunk._is_cscalar: if thunk.dtype._is_udt: + # NOT COVERED dtype_name = "UDT" thunk = _Pointer(thunk) else: diff --git a/graphblas/core/operator/agg.py b/graphblas/core/operator/agg.py index 036149b1f..09d644c32 100644 --- a/graphblas/core/operator/agg.py +++ b/graphblas/core/operator/agg.py @@ -5,6 +5,7 @@ from ... import agg, backend, binary, monoid, semiring, unary from ...dtypes import INT64, lookup_dtype +from .. import _supports_udfs from ..utils import output_type @@ -38,6 +39,7 @@ def __init__( semiring=None, switch=False, semiring2=None, + applybegin=None, finalize=None, composite=None, custom=None, @@ -52,6 +54,7 @@ def __init__( self._semiring = semiring self._semiring2 = semiring2 self._switch = switch + self._applybegin = applybegin self._finalize = finalize self._composite = composite self._custom = custom @@ -152,8 +155,11 @@ def __repr__(self): def _new(self, updater, expr, *, in_composite=False): agg = self.parent + opts = updater.opts if agg._monoid is not None: x = expr.args[0] + if agg._applybegin is not None: # pragma: no cover (unused) + x = agg._applybegin(x).new(**opts) method = getattr(x, expr.method_name) if expr.output_type.__name__ == "Scalar": expr = method(agg._monoid[self.type], allow_empty=not expr._is_cscalar) @@ -167,7 +173,6 @@ def _new(self, updater, expr, *, in_composite=False): return parent._as_vector() return - opts = updater.opts if agg._composite is not None: # Masks are applied throughout the aggregation, including composite aggregations. # Aggregations done while `in_composite is True` should return the updater parent @@ -203,6 +208,8 @@ def _new(self, updater, expr, *, in_composite=False): if expr.cfunc_name == "GrB_Matrix_reduce_Aggregator": # Matrix -> Vector A = expr.args[0] + if agg._applybegin is not None: + A = agg._applybegin(A).new(**opts) orig_updater = updater if agg._finalize is not None: step1 = expr.construct_output(semiring.return_type) @@ -223,6 +230,8 @@ def _new(self, updater, expr, *, in_composite=False): elif expr.cfunc_name.startswith("GrB_Vector_reduce"): # Vector -> Scalar v = expr.args[0] + if agg._applybegin is not None: + v = agg._applybegin(v).new(**opts) step1 = expr._new_vector(semiring.return_type, size=1) init = expr._new_matrix(agg._initdtype, nrows=v._size, ncols=1) init(**opts)[...] = agg._initval # O(1) dense column vector in SuiteSparse 5 @@ -242,6 +251,8 @@ def _new(self, updater, expr, *, in_composite=False): elif expr.cfunc_name.startswith("GrB_Matrix_reduce"): # Matrix -> Scalar A = expr.args[0] + if agg._applybegin is not None: + A = agg._applybegin(A).new(**opts) # We need to compute in two steps: Matrix -> Vector -> Scalar. # This has not been benchmarked or optimized. # We may be able to intelligently choose the faster path. @@ -339,11 +350,21 @@ def __reduce__(self): # logaddexp2 = Aggregator('logaddexp2', monoid=semiring.numpy.logaddexp2) # hypot as monoid doesn't work if single negative element! # hypot = Aggregator('hypot', monoid=semiring.numpy.hypot) +# hypot = Aggregator('hypot', applybegin=unary.abs, monoid=semiring.numpy.hypot) agg.L0norm = agg.count_nonzero -agg.L1norm = Aggregator("L1norm", semiring="plus_absfirst", semiring2=semiring.plus_first) agg.L2norm = agg.hypot -agg.Linfnorm = Aggregator("Linfnorm", semiring="max_absfirst", semiring2=semiring.max_first) +if _supports_udfs: + agg.L1norm = Aggregator("L1norm", semiring="plus_absfirst", semiring2=semiring.plus_first) + agg.Linfnorm = Aggregator("Linfnorm", semiring="max_absfirst", semiring2=semiring.max_first) +else: + # Are these always better? + agg.L1norm = Aggregator( + "L1norm", applybegin=unary.abs, semiring=semiring.plus_first, semiring2=semiring.plus_first + ) + agg.Linfnorm = Aggregator( + "Linfnorm", applybegin=unary.abs, semiring=semiring.max_first, semiring2=semiring.max_first + ) # Composite diff --git a/graphblas/core/operator/base.py b/graphblas/core/operator/base.py index 38a76cbcf..a40438f14 100644 --- a/graphblas/core/operator/base.py +++ b/graphblas/core/operator/base.py @@ -1,16 +1,19 @@ -from functools import lru_cache, reduce -from operator import getitem, mul +from functools import lru_cache +from operator import getitem from types import BuiltinFunctionType, ModuleType -import numba -import numpy as np - from ... import _STANDARD_OPERATOR_NAMES, backend, op from ...dtypes import BOOL, INT8, UINT64, _supports_complex, lookup_dtype -from .. import lib +from .. import _has_numba, _supports_udfs, lib from ..expr import InfixExprBase from ..utils import output_type +if _has_numba: + import numba + from numba import NumbaError +else: + NumbaError = TypeError + UNKNOWN_OPCLASS = "UnknownOpClass" # These now live as e.g. `gb.unary.ss.positioni` @@ -158,96 +161,69 @@ def _call_op(op, left, right=None, thunk=None, **kwargs): ) -_udt_mask_cache = {} - - -def _udt_mask(dtype): - """Create mask to determine which bytes of UDTs to use for equality check.""" - if dtype in _udt_mask_cache: - return _udt_mask_cache[dtype] - if dtype.subdtype is not None: - mask = _udt_mask(dtype.subdtype[0]) - N = reduce(mul, dtype.subdtype[1]) - rv = np.concatenate([mask] * N) - elif dtype.names is not None: - prev_offset = mask = None - masks = [] - for name in dtype.names: - dtype2, offset = dtype.fields[name] - if mask is not None: - masks.append(np.pad(mask, (0, offset - prev_offset - mask.size))) - mask = _udt_mask(dtype2) - prev_offset = offset - masks.append(np.pad(mask, (0, dtype.itemsize - prev_offset - mask.size))) - rv = np.concatenate(masks) - else: - rv = np.ones(dtype.itemsize, dtype=bool) - # assert rv.size == dtype.itemsize - _udt_mask_cache[dtype] = rv - return rv - - -def _get_udt_wrapper(numba_func, return_type, dtype, dtype2=None, *, include_indexes=False): - ztype = INT8 if return_type == BOOL else return_type - xtype = INT8 if dtype == BOOL else dtype - nt = numba.types - wrapper_args = [nt.CPointer(ztype.numba_type), nt.CPointer(xtype.numba_type)] - if include_indexes: - wrapper_args.extend([UINT64.numba_type, UINT64.numba_type]) - if dtype2 is not None: - ytype = INT8 if dtype2 == BOOL else dtype2 - wrapper_args.append(nt.CPointer(ytype.numba_type)) - wrapper_sig = nt.void(*wrapper_args) - - zarray = xarray = yarray = BL = BR = yarg = yname = rcidx = "" - if return_type._is_udt: - if return_type.np_type.subdtype is None: - zarray = " z = numba.carray(z_ptr, 1)\n" - zname = "z[0]" +if _has_numba: + + def _get_udt_wrapper(numba_func, return_type, dtype, dtype2=None, *, include_indexes=False): + ztype = INT8 if return_type == BOOL else return_type + xtype = INT8 if dtype == BOOL else dtype + nt = numba.types + wrapper_args = [nt.CPointer(ztype.numba_type), nt.CPointer(xtype.numba_type)] + if include_indexes: + wrapper_args.extend([UINT64.numba_type, UINT64.numba_type]) + if dtype2 is not None: + ytype = INT8 if dtype2 == BOOL else dtype2 + wrapper_args.append(nt.CPointer(ytype.numba_type)) + wrapper_sig = nt.void(*wrapper_args) + + zarray = xarray = yarray = BL = BR = yarg = yname = rcidx = "" + if return_type._is_udt: + if return_type.np_type.subdtype is None: + zarray = " z = numba.carray(z_ptr, 1)\n" + zname = "z[0]" + else: + zname = "z_ptr[0]" + BR = "[0]" else: zname = "z_ptr[0]" - BR = "[0]" - else: - zname = "z_ptr[0]" - if return_type == BOOL: - BL = "bool(" - BR = ")" - - if dtype._is_udt: - if dtype.np_type.subdtype is None: - xarray = " x = numba.carray(x_ptr, 1)\n" - xname = "x[0]" - else: - xname = "x_ptr" - elif dtype == BOOL: - xname = "bool(x_ptr[0])" - else: - xname = "x_ptr[0]" - - if dtype2 is not None: - yarg = ", y_ptr" - if dtype2._is_udt: - if dtype2.np_type.subdtype is None: - yarray = " y = numba.carray(y_ptr, 1)\n" - yname = ", y[0]" + if return_type == BOOL: + BL = "bool(" + BR = ")" + + if dtype._is_udt: + if dtype.np_type.subdtype is None: + xarray = " x = numba.carray(x_ptr, 1)\n" + xname = "x[0]" else: - yname = ", y_ptr" - elif dtype2 == BOOL: - yname = ", bool(y_ptr[0])" + xname = "x_ptr" + elif dtype == BOOL: + xname = "bool(x_ptr[0])" else: - yname = ", y_ptr[0]" + xname = "x_ptr[0]" + + if dtype2 is not None: + yarg = ", y_ptr" + if dtype2._is_udt: + if dtype2.np_type.subdtype is None: + yarray = " y = numba.carray(y_ptr, 1)\n" + yname = ", y[0]" + else: + yname = ", y_ptr" + elif dtype2 == BOOL: + yname = ", bool(y_ptr[0])" + else: + yname = ", y_ptr[0]" - if include_indexes: - rcidx = ", row, col" + if include_indexes: + rcidx = ", row, col" - d = {"numba": numba, "numba_func": numba_func} - text = ( - f"def wrapper(z_ptr, x_ptr{rcidx}{yarg}):\n" - f"{zarray}{xarray}{yarray}" - f" {zname} = {BL}numba_func({xname}{rcidx}{yname}){BR}\n" - ) - exec(text, d) # pylint: disable=exec-used - return d["wrapper"], wrapper_sig + d = {"numba": numba, "numba_func": numba_func} + text = ( + f"def wrapper(z_ptr, x_ptr{rcidx}{yarg}):\n" + f"{zarray}{xarray}{yarray}" + f" {zname} = {BL}numba_func({xname}{rcidx}{yname}){BR}\n" + ) + exec(text, d) # pylint: disable=exec-used + return d["wrapper"], wrapper_sig class TypedOpBase: @@ -360,6 +336,8 @@ def __getitem__(self, type_): raise KeyError(f"{self.name} does not work with {type_}") else: return self._typed_ops[type_] + if not _supports_udfs: + raise KeyError(f"{self.name} does not work with {type_}") # This is a UDT or is able to operate on UDTs such as `first` any `any` dtype = lookup_dtype(type_) return self._compile_udt(dtype, dtype) @@ -376,7 +354,7 @@ def __delitem__(self, type_): def __contains__(self, type_): try: self[type_] - except (TypeError, KeyError, numba.NumbaError): + except (TypeError, KeyError, NumbaError): return False return True @@ -487,7 +465,7 @@ def _initialize(cls, include_in_ops=True): if type_ is None: type_ = BOOL else: - if type_ is None: # pragma: no cover + if type_ is None: # pragma: no cover (safety) raise TypeError(f"Unable to determine return type for {varname}") if return_prefix is None: return_type = type_ @@ -513,6 +491,13 @@ def _deserialize(cls, name, *args): return rv # Should we verify this is what the user expects? return cls.register_new(name, *args) + @classmethod + def _check_supports_udf(cls, method_name): + if not _supports_udfs: + raise RuntimeError( + f"{cls.__name__}.{method_name}(...) unavailable; install numba for UDF support" + ) + _builtin_to_op = {} # Populated in .utils diff --git a/graphblas/core/operator/binary.py b/graphblas/core/operator/binary.py index eeb72ea3b..8d41a097e 100644 --- a/graphblas/core/operator/binary.py +++ b/graphblas/core/operator/binary.py @@ -1,9 +1,9 @@ import inspect import re -from functools import lru_cache +from functools import lru_cache, reduce +from operator import mul from types import FunctionType -import numba import numpy as np from ... import _STANDARD_OPERATOR_NAMES, backend, binary, monoid, op @@ -24,7 +24,7 @@ lookup_dtype, ) from ...exceptions import UdfParseError, check_status_carg -from .. import ffi, lib +from .. import _has_numba, _supports_udfs, ffi, lib from ..expr import InfixExprBase from .base import ( _SS_OPERATORS, @@ -33,16 +33,46 @@ TypedOpBase, _call_op, _deserialize_parameterized, - _get_udt_wrapper, _hasop, - _udt_mask, ) +if _has_numba: + import numba + + from .base import _get_udt_wrapper if _supports_complex: from ...dtypes import FC32, FC64 ffi_new = ffi.new +if _has_numba: + _udt_mask_cache = {} + + def _udt_mask(dtype): + """Create mask to determine which bytes of UDTs to use for equality check.""" + if dtype in _udt_mask_cache: + return _udt_mask_cache[dtype] + if dtype.subdtype is not None: + mask = _udt_mask(dtype.subdtype[0]) + N = reduce(mul, dtype.subdtype[1]) + rv = np.concatenate([mask] * N) + elif dtype.names is not None: + prev_offset = mask = None + masks = [] + for name in dtype.names: + dtype2, offset = dtype.fields[name] + if mask is not None: + masks.append(np.pad(mask, (0, offset - prev_offset - mask.size))) + mask = _udt_mask(dtype2) + prev_offset = offset + masks.append(np.pad(mask, (0, dtype.itemsize - prev_offset - mask.size))) + rv = np.concatenate(masks) + else: + rv = np.ones(dtype.itemsize, dtype=bool) + # assert rv.size == dtype.itemsize + _udt_mask_cache[dtype] = rv + return rv + class TypedBuiltinBinaryOp(TypedOpBase): __slots__ = () @@ -601,6 +631,7 @@ def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=Fals Because it is not registered in the namespace, the name is optional. """ + cls._check_supports_udf("register_anonymous") if parameterized: return ParameterizedBinaryOp(name, func, anonymous=True, is_udt=is_udt) return cls._build(name, func, anonymous=True, is_udt=is_udt) @@ -621,6 +652,7 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal >>> dir(gb.binary) [..., 'max_zero', ...] """ + cls._check_supports_udf("register_new") module, funcname = cls._remove_nesting(name) if lazy: module._delayed[funcname] = ( @@ -681,21 +713,22 @@ def _initialize(cls): orig_op.gb_name, ) new_op._add(cur_op) - # Add floordiv - # cdiv truncates towards 0, while floordiv truncates towards -inf - BinaryOp.register_new("floordiv", _floordiv, lazy=True) # cast to integer - BinaryOp.register_new("rfloordiv", _rfloordiv, lazy=True) # cast to integer + if _supports_udfs: + # Add floordiv + # cdiv truncates towards 0, while floordiv truncates towards -inf + BinaryOp.register_new("floordiv", _floordiv, lazy=True) # cast to integer + BinaryOp.register_new("rfloordiv", _rfloordiv, lazy=True) # cast to integer - # For aggregators - BinaryOp.register_new("absfirst", _absfirst, lazy=True) - BinaryOp.register_new("abssecond", _abssecond, lazy=True) - BinaryOp.register_new("rpow", _rpow, lazy=True) + # For aggregators + BinaryOp.register_new("absfirst", _absfirst, lazy=True) + BinaryOp.register_new("abssecond", _abssecond, lazy=True) + BinaryOp.register_new("rpow", _rpow, lazy=True) - # For algorithms - binary._delayed["binom"] = (_register_binom, {}) # Lazy with custom creation - op._delayed["binom"] = binary + # For algorithms + binary._delayed["binom"] = (_register_binom, {}) # Lazy with custom creation + op._delayed["binom"] = binary - BinaryOp.register_new("isclose", _isclose, parameterized=True) + BinaryOp.register_new("isclose", _isclose, parameterized=True) # Update type information with sane coercion position_dtypes = [ @@ -777,14 +810,23 @@ def _initialize(cls): if right_name not in binary._delayed: if right_name in _SS_OPERATORS: right = binary._deprecated[right_name] - else: + elif _supports_udfs: right = getattr(binary, right_name) + else: + right = getattr(binary, right_name, None) + if right is None: + continue if backend == "suitesparse" and left_name in _SS_OPERATORS: right._commutes_to = f"ss.{left_name}" else: right._commutes_to = left_name for name in cls._commutative: - cur_op = getattr(binary, name) + if _supports_udfs: + cur_op = getattr(binary, name) + else: + cur_op = getattr(binary, name, None) + if cur_op is None: + continue cur_op._commutes_to = name for left_name, right_name in cls._commutes_to_in_semiring.items(): if left_name in _SS_OPERATORS: @@ -805,7 +847,10 @@ def _initialize(cls): (binary.any, _first), ]: binop.orig_func = func - binop._numba_func = numba.njit(func) + if _has_numba: + binop._numba_func = numba.njit(func) + else: + binop._numba_func = None binop._udt_types = {} binop._udt_ops = {} binary.any._numba_func = binary.first._numba_func diff --git a/graphblas/core/operator/indexunary.py b/graphblas/core/operator/indexunary.py index 5fdafb62a..ad5d841d0 100644 --- a/graphblas/core/operator/indexunary.py +++ b/graphblas/core/operator/indexunary.py @@ -2,21 +2,16 @@ import re from types import FunctionType -import numba - from ... import _STANDARD_OPERATOR_NAMES, indexunary, select from ...dtypes import BOOL, FP64, INT8, INT64, UINT64, _sample_values, lookup_dtype from ...exceptions import UdfParseError, check_status_carg -from .. import ffi, lib -from .base import ( - OpBase, - ParameterizedUdf, - TypedOpBase, - _call_op, - _deserialize_parameterized, - _get_udt_wrapper, -) +from .. import _has_numba, ffi, lib +from .base import OpBase, ParameterizedUdf, TypedOpBase, _call_op, _deserialize_parameterized + +if _has_numba: + import numba + from .base import _get_udt_wrapper ffi_new = ffi.new @@ -65,6 +60,7 @@ def _call(self, *args, **kwargs): return IndexUnaryOp.register_anonymous(indexunary, self.name, is_udt=self._is_udt) def __reduce__(self): + # NOT COVERED name = f"indexunary.{self.name}" if not self._anonymous and name in _STANDARD_OPERATOR_NAMES: return name @@ -72,6 +68,7 @@ def __reduce__(self): @staticmethod def _deserialize(name, func, anonymous): + # NOT COVERED if anonymous: return IndexUnaryOp.register_anonymous(func, name, parameterized=True) if (rv := IndexUnaryOp._find(name)) is not None: @@ -249,6 +246,7 @@ def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=Fals Because it is not registered in the namespace, the name is optional. """ + cls._check_supports_udf("register_anonymous") if parameterized: return ParameterizedIndexUnaryOp(name, func, anonymous=True, is_udt=is_udt) return cls._build(name, func, anonymous=True, is_udt=is_udt) @@ -265,6 +263,7 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal >>> dir(gb.indexunary) [..., 'row_mod', ...] """ + cls._check_supports_udf("register_new") module, funcname = cls._remove_nesting(name) if lazy: module._delayed[funcname] = ( @@ -281,9 +280,12 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal if all(x == BOOL for x in indexunary_op.types.values()): from .select import SelectOp - setattr(select, funcname, SelectOp._from_indexunary(indexunary_op)) + select_module, funcname = SelectOp._remove_nesting(name, strict=False) + setattr(select_module, funcname, SelectOp._from_indexunary(indexunary_op)) + if not cls._initialized: # pragma: no cover (safety) + _STANDARD_OPERATOR_NAMES.add(f"{SelectOp._modname}.{name}") - if not cls._initialized: + if not cls._initialized: # pragma: no cover (safety) _STANDARD_OPERATOR_NAMES.add(f"{cls._modname}.{name}") if not lazy: return indexunary_op @@ -323,6 +325,7 @@ def _initialize(cls): "valueeq", "valuene", "valuegt", "valuege", "valuelt", "valuele"]: iop = getattr(indexunary, name) setattr(select, name, SelectOp._from_indexunary(iop)) + _STANDARD_OPERATOR_NAMES.add(f"{SelectOp._modname}.{name}") # fmt: on cls._initialized = True @@ -348,10 +351,12 @@ def __init__( def __reduce__(self): if self._anonymous: if hasattr(self.orig_func, "_parameterized_info"): + # NOT COVERED return (_deserialize_parameterized, self.orig_func._parameterized_info) return (self.register_anonymous, (self.orig_func, self.name)) if (name := f"indexunary.{self.name}") in _STANDARD_OPERATOR_NAMES: return name + # NOT COVERED return (self._deserialize, (self.name, self.orig_func)) __call__ = TypedBuiltinIndexUnaryOp.__call__ diff --git a/graphblas/core/operator/select.py b/graphblas/core/operator/select.py index 844565f3a..27567eb2f 100644 --- a/graphblas/core/operator/select.py +++ b/graphblas/core/operator/select.py @@ -51,6 +51,7 @@ def _call(self, *args, **kwargs): return SelectOp.register_anonymous(sel, self.name, is_udt=self._is_udt) def __reduce__(self): + # NOT COVERED name = f"select.{self.name}" if not self._anonymous and name in _STANDARD_OPERATOR_NAMES: return name @@ -58,6 +59,7 @@ def __reduce__(self): @staticmethod def _deserialize(name, func, anonymous): + # NOT COVERED if anonymous: return SelectOp.register_anonymous(func, name, parameterized=True) if (rv := SelectOp._find(name)) is not None: @@ -124,6 +126,7 @@ def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=Fals Because it is not registered in the namespace, the name is optional. """ + cls._check_supports_udf("register_anonymous") if parameterized: return ParameterizedSelectOp(name, func, anonymous=True, is_udt=is_udt) iop = IndexUnaryOp._build(name, func, anonymous=True, is_udt=is_udt) @@ -140,13 +143,36 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal >>> dir(gb.select) [..., 'upper_left_triangle', ...] """ + cls._check_supports_udf("register_new") iop = IndexUnaryOp.register_new( name, func, parameterized=parameterized, is_udt=is_udt, lazy=lazy ) + module, funcname = cls._remove_nesting(name, strict=False) + if lazy: + module._delayed[funcname] = ( + cls._get_delayed, + {"name": name}, + ) + elif parameterized: + op = ParameterizedSelectOp(funcname, func, is_udt=is_udt) + setattr(module, funcname, op) + return op + elif not all(x == BOOL for x in iop.types.values()): + # Undo registration of indexunaryop + imodule, funcname = IndexUnaryOp._remove_nesting(name, strict=False) + delattr(imodule, funcname) + raise ValueError("SelectOp must have BOOL return type") + else: + return getattr(module, funcname) + + @classmethod + def _get_delayed(cls, name): + imodule, funcname = IndexUnaryOp._remove_nesting(name, strict=False) + iop = getattr(imodule, name) if not all(x == BOOL for x in iop.types.values()): raise ValueError("SelectOp must have BOOL return type") - if lazy: - return getattr(select, iop.name) + module, funcname = cls._remove_nesting(name, strict=False) + return getattr(module, funcname) @classmethod def _initialize(cls): @@ -172,16 +198,19 @@ def __init__( self.is_positional = is_positional self._is_udt = is_udt if is_udt: + # NOT COVERED self._udt_types = {} # {dtype: DataType} self._udt_ops = {} # {dtype: TypedUserIndexUnaryOp} def __reduce__(self): if self._anonymous: if hasattr(self.orig_func, "_parameterized_info"): + # NOT COVERED return (_deserialize_parameterized, self.orig_func._parameterized_info) return (self.register_anonymous, (self.orig_func, self.name)) if (name := f"select.{self.name}") in _STANDARD_OPERATOR_NAMES: return name + # NOT COVERED return (self._deserialize, (self.name, self.orig_func)) __call__ = TypedBuiltinSelectOp.__call__ diff --git a/graphblas/core/operator/semiring.py b/graphblas/core/operator/semiring.py index 06450e007..ac716b9dd 100644 --- a/graphblas/core/operator/semiring.py +++ b/graphblas/core/operator/semiring.py @@ -17,7 +17,7 @@ _supports_complex, ) from ...exceptions import check_status_carg -from .. import ffi, lib +from .. import _supports_udfs, ffi, lib from .base import _SS_OPERATORS, OpBase, ParameterizedUdf, TypedOpBase, _call_op, _hasop from .binary import BinaryOp, ParameterizedBinaryOp from .monoid import Monoid, ParameterizedMonoid @@ -358,15 +358,17 @@ def _initialize(cls): for orig_name, orig in div_semirings.items(): cls.register_new(f"{orig_name[:-3]}truediv", orig.monoid, binary.truediv, lazy=True) cls.register_new(f"{orig_name[:-3]}rtruediv", orig.monoid, "rtruediv", lazy=True) - cls.register_new(f"{orig_name[:-3]}floordiv", orig.monoid, "floordiv", lazy=True) - cls.register_new(f"{orig_name[:-3]}rfloordiv", orig.monoid, "rfloordiv", lazy=True) + if _supports_udfs: + cls.register_new(f"{orig_name[:-3]}floordiv", orig.monoid, "floordiv", lazy=True) + cls.register_new(f"{orig_name[:-3]}rfloordiv", orig.monoid, "rfloordiv", lazy=True) # For aggregators cls.register_new("plus_pow", monoid.plus, binary.pow) - cls.register_new("plus_rpow", monoid.plus, "rpow", lazy=True) - cls.register_new("plus_absfirst", monoid.plus, "absfirst", lazy=True) - cls.register_new("max_absfirst", monoid.max, "absfirst", lazy=True) - cls.register_new("plus_abssecond", monoid.plus, "abssecond", lazy=True) - cls.register_new("max_abssecond", monoid.max, "abssecond", lazy=True) + if _supports_udfs: + cls.register_new("plus_rpow", monoid.plus, "rpow", lazy=True) + cls.register_new("plus_absfirst", monoid.plus, "absfirst", lazy=True) + cls.register_new("max_absfirst", monoid.max, "absfirst", lazy=True) + cls.register_new("plus_abssecond", monoid.plus, "abssecond", lazy=True) + cls.register_new("max_abssecond", monoid.max, "abssecond", lazy=True) # Update type information with sane coercion for lname in ["any", "eq", "land", "lor", "lxnor", "lxor"]: diff --git a/graphblas/core/operator/unary.py b/graphblas/core/operator/unary.py index 6b1319057..1432a9387 100644 --- a/graphblas/core/operator/unary.py +++ b/graphblas/core/operator/unary.py @@ -2,8 +2,6 @@ import re from types import FunctionType -import numba - from ... import _STANDARD_OPERATOR_NAMES, op, unary from ...dtypes import ( BOOL, @@ -22,7 +20,7 @@ lookup_dtype, ) from ...exceptions import UdfParseError, check_status_carg -from .. import ffi, lib +from .. import _has_numba, ffi, lib from ..utils import output_type from .base import ( _SS_OPERATORS, @@ -30,12 +28,15 @@ ParameterizedUdf, TypedOpBase, _deserialize_parameterized, - _get_udt_wrapper, _hasop, ) if _supports_complex: from ...dtypes import FC32, FC64 +if _has_numba: + import numba + + from .base import _get_udt_wrapper ffi_new = ffi.new @@ -276,6 +277,7 @@ def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=Fals Because it is not registered in the namespace, the name is optional. """ + cls._check_supports_udf("register_anonymous") if parameterized: return ParameterizedUnaryOp(name, func, anonymous=True, is_udt=is_udt) return cls._build(name, func, anonymous=True, is_udt=is_udt) @@ -289,6 +291,7 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal >>> dir(gb.unary) [..., 'plus_one', ...] """ + cls._check_supports_udf("register_new") module, funcname = cls._remove_nesting(name) if lazy: module._delayed[funcname] = ( @@ -372,7 +375,10 @@ def _initialize(cls): (unary.one, _one), ]: unop.orig_func = func - unop._numba_func = numba.njit(func) + if _has_numba: + unop._numba_func = numba.njit(func) + else: + unop._numba_func = None unop._udt_types = {} unop._udt_ops = {} cls._initialized = True diff --git a/graphblas/core/recorder.py b/graphblas/core/recorder.py index ce79c85ff..2268c31eb 100644 --- a/graphblas/core/recorder.py +++ b/graphblas/core/recorder.py @@ -137,10 +137,10 @@ def _repr_base_(self): tail = "\n\n\n" return "\n".join(head), tail - def _repr_html_(self): # pragma: no cover + def _repr_html_(self): try: from IPython.display import Code - except ImportError as exc: + except ImportError as exc: # pragma: no cover (import) raise NotImplementedError from exc lines = self._get_repr_lines() code = Code("\n".join(lines), language="C") diff --git a/graphblas/core/scalar.py b/graphblas/core/scalar.py index 93b5ebb4b..a7a251a1d 100644 --- a/graphblas/core/scalar.py +++ b/graphblas/core/scalar.py @@ -3,15 +3,19 @@ import numpy as np from .. import backend, binary, config, monoid -from ..binary import isclose from ..dtypes import _INDEX, FP64, lookup_dtype, unify from ..exceptions import EmptyObject, check_status -from . import automethods, ffi, lib, utils +from . import _has_numba, _supports_udfs, automethods, ffi, lib, utils from .base import BaseExpression, BaseType, call from .expr import AmbiguousAssignOrExtract from .operator import get_typed_op from .utils import _Pointer, output_type, wrapdoc +if _supports_udfs: + from ..binary import isclose +else: + from .operator.binary import _isclose as isclose + ffi_new = ffi.new @@ -261,6 +265,17 @@ def isclose(self, other, *, rel_tol=1e-7, abs_tol=0.0, check_dtype=False): return False # We can't yet call a UDF on a scalar as part of the spec, so let's do it ourselves isclose_func = isclose(rel_tol, abs_tol) + if not _has_numba: + # Check if types are compatible + get_typed_op( + binary.eq, + self.dtype, + other.dtype, + is_left_scalar=True, + is_right_scalar=True, + kind="binary", + ) + return isclose_func(self.value, other.value) isclose_func = get_typed_op( isclose_func, self.dtype, diff --git a/graphblas/core/ss/matrix.py b/graphblas/core/ss/matrix.py index b1869f198..cac0296c7 100644 --- a/graphblas/core/ss/matrix.py +++ b/graphblas/core/ss/matrix.py @@ -1,9 +1,7 @@ import itertools import warnings -import numba import numpy as np -from numba import njit from suitesparse_graphblas.utils import claim_buffer, claim_buffer_2d, unclaim_buffer import graphblas as gb @@ -11,7 +9,7 @@ from ... import binary, monoid from ...dtypes import _INDEX, BOOL, INT64, UINT64, _string_to_dtype, lookup_dtype from ...exceptions import _error_code_lookup, check_status, check_status_carg -from .. import NULL, ffi, lib +from .. import NULL, _has_numba, ffi, lib from ..base import call from ..operator import get_typed_op from ..scalar import Scalar, _as_scalar, _scalar_index @@ -30,6 +28,16 @@ from .config import BaseConfig from .descriptor import get_descriptor +if _has_numba: + from numba import njit, prange +else: + + def njit(func=None, **kwargs): + if func is not None: + return func + return njit + + prange = range ffi_new = ffi.new @@ -888,7 +896,7 @@ def _export(self, format=None, *, sort=False, give_ownership=False, raw=False, m col_indices = claim_buffer(ffi, Aj[0], Aj_size[0] // index_dtype.itemsize, index_dtype) values = claim_buffer(ffi, Ax[0], Ax_size[0] // dtype.itemsize, dtype) if not raw: - if indptr.size > nrows + 1: + if indptr.size > nrows + 1: # pragma: no cover (suitesparse) indptr = indptr[: nrows + 1] if col_indices.size > nvals: col_indices = col_indices[:nvals] @@ -929,7 +937,7 @@ def _export(self, format=None, *, sort=False, give_ownership=False, raw=False, m row_indices = claim_buffer(ffi, Ai[0], Ai_size[0] // index_dtype.itemsize, index_dtype) values = claim_buffer(ffi, Ax[0], Ax_size[0] // dtype.itemsize, dtype) if not raw: - if indptr.size > ncols + 1: + if indptr.size > ncols + 1: # pragma: no cover (suitesparse) indptr = indptr[: ncols + 1] if row_indices.size > nvals: row_indices = row_indices[:nvals] @@ -1786,6 +1794,7 @@ def import_hypercsc( ---------- nrows : int ncols : int + cols : array-like indptr : array-like values : array-like row_indices : array-like @@ -4371,28 +4380,28 @@ def deserialize(cls, data, dtype=None, *, name=None, **opts): return rv -@numba.njit(parallel=True) +@njit(parallel=True) def argsort_values(indptr, indices, values): # pragma: no cover (numba) rv = np.empty(indptr[-1], dtype=np.uint64) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): rv[indptr[i] : indptr[i + 1]] = indices[ np.int64(indptr[i]) + np.argsort(values[indptr[i] : indptr[i + 1]]) ] return rv -@numba.njit(parallel=True) +@njit(parallel=True) def sort_values(indptr, values): # pragma: no cover (numba) rv = np.empty(indptr[-1], dtype=values.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): rv[indptr[i] : indptr[i + 1]] = np.sort(values[indptr[i] : indptr[i + 1]]) return rv -@numba.njit(parallel=True) +@njit(parallel=True) def compact_values(old_indptr, new_indptr, values): # pragma: no cover (numba) rv = np.empty(new_indptr[-1], dtype=values.dtype) - for i in numba.prange(new_indptr.size - 1): + for i in prange(new_indptr.size - 1): start = np.int64(new_indptr[i]) offset = np.int64(old_indptr[i]) - start for j in range(start, new_indptr[i + 1]): @@ -4400,17 +4409,17 @@ def compact_values(old_indptr, new_indptr, values): # pragma: no cover (numba) return rv -@numba.njit(parallel=True) +@njit(parallel=True) def reverse_values(indptr, values): # pragma: no cover (numba) rv = np.empty(indptr[-1], dtype=values.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): offset = np.int64(indptr[i]) + np.int64(indptr[i + 1]) - 1 for j in range(indptr[i], indptr[i + 1]): rv[j] = values[offset - j] return rv -@numba.njit(parallel=True) +@njit(parallel=True) def compact_indices(indptr, k): # pragma: no cover (numba) """Given indptr from hypercsr, create a new col_indices array that is compact. @@ -4420,7 +4429,7 @@ def compact_indices(indptr, k): # pragma: no cover (numba) indptr = create_indptr(indptr, k) col_indices = np.empty(indptr[-1], dtype=np.uint64) N = np.int64(0) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): start = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - start N = max(N, deg) @@ -4433,7 +4442,7 @@ def compact_indices(indptr, k): # pragma: no cover (numba) def choose_random1(indptr): # pragma: no cover (numba) choices = np.empty(indptr.size - 1, dtype=indptr.dtype) new_indptr = np.arange(indptr.size, dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): idx = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - idx if deg == 1: @@ -4470,7 +4479,7 @@ def choose_random(indptr, k): # pragma: no cover (numba) # be nice to have them sorted if convenient to do so. new_indptr = create_indptr(indptr, k) choices = np.empty(new_indptr[-1], dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): idx = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - idx if k < deg: @@ -4551,7 +4560,7 @@ def choose_first(indptr, k): # pragma: no cover (numba) new_indptr = create_indptr(indptr, k) choices = np.empty(new_indptr[-1], dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): idx = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - idx if k < deg: @@ -4575,7 +4584,7 @@ def choose_last(indptr, k): # pragma: no cover (numba) new_indptr = create_indptr(indptr, k) choices = np.empty(new_indptr[-1], dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): idx = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - idx if k < deg: @@ -4608,19 +4617,20 @@ def indices_to_indptr(indices, size): # pragma: no cover (numba) """Calculate the indptr for e.g. CSR from sorted COO rows.""" indptr = np.zeros(size, dtype=indices.dtype) index = np.uint64(0) + one = np.uint64(1) for i in range(indices.size): row = indices[i] if row != index: - indptr[index + 1] = i + indptr[index + one] = i index = row - indptr[index + 1] = indices.size + indptr[index + one] = indices.size return indptr @njit(parallel=True) def indptr_to_indices(indptr): # pragma: no cover (numba) indices = np.empty(indptr[-1], dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): for j in range(indptr[i], indptr[i + 1]): indices[j] = i return indices diff --git a/graphblas/core/ss/vector.py b/graphblas/core/ss/vector.py index 343335773..2b1e8bf05 100644 --- a/graphblas/core/ss/vector.py +++ b/graphblas/core/ss/vector.py @@ -1,7 +1,6 @@ import itertools import numpy as np -from numba import njit from suitesparse_graphblas.utils import claim_buffer, unclaim_buffer import graphblas as gb @@ -23,7 +22,7 @@ ) from .config import BaseConfig from .descriptor import get_descriptor -from .matrix import _concat_mn +from .matrix import _concat_mn, njit from .prefix_scan import prefix_scan ffi_new = ffi.new @@ -588,7 +587,7 @@ def _export(self, format=None, *, sort=False, give_ownership=False, raw=False, m if is_iso: if values.size > 1: # pragma: no cover (suitesparse) values = values[:1] - elif values.size > size: # pragma: no branch (suitesparse) + elif values.size > size: # pragma: no cover (suitesparse) values = values[:size] rv = { "bitmap": bitmap, diff --git a/graphblas/core/utils.py b/graphblas/core/utils.py index 0beeb4a2a..77c64a7ac 100644 --- a/graphblas/core/utils.py +++ b/graphblas/core/utils.py @@ -131,6 +131,7 @@ def get_shape(nrows, ncols, dtype=None, **arrays): # We could be smarter and determine the shape of the dtype sub-arrays if arr.ndim >= 3: break + # BRANCH NOT COVERED elif arr.ndim == 2: break else: diff --git a/graphblas/core/vector.py b/graphblas/core/vector.py index 8231691c6..57851420d 100644 --- a/graphblas/core/vector.py +++ b/graphblas/core/vector.py @@ -3,10 +3,10 @@ import numpy as np -from .. import backend, binary, monoid, select, semiring +from .. import backend, binary, monoid, select, semiring, unary from ..dtypes import _INDEX, FP64, INT64, lookup_dtype, unify from ..exceptions import DimensionMismatch, NoValue, check_status -from . import automethods, ffi, lib, utils +from . import _supports_udfs, automethods, ffi, lib, utils from .base import BaseExpression, BaseType, _check_mask, call from .descriptor import lookup as descriptor_lookup from .expr import _ALL_INDICES, AmbiguousAssignOrExtract, IndexerResolver, Updater @@ -93,6 +93,45 @@ def _select_mask(updater, obj, mask): updater << obj.dup(mask=mask) +def _isclose_recipe(self, other, rel_tol, abs_tol, **opts): + # x == y or abs(x - y) <= max(rel_tol * max(abs(x), abs(y)), abs_tol) + isequal = self.ewise_mult(other, binary.eq).new(bool, name="isclose", **opts) + if isequal._nvals != self._nvals: + return False + if type(isequal) is Vector: + val = isequal.reduce(monoid.land, allow_empty=False).new(**opts).value + else: + val = isequal.reduce_scalar(monoid.land, allow_empty=False).new(**opts).value + if val: + return True + # So we can use structural mask below + isequal(**opts) << select.value(isequal == True) # noqa: E712 + + # abs(x) + x = self.apply(unary.abs).new(FP64, mask=~isequal.S, **opts) + # abs(y) + y = other.apply(unary.abs).new(FP64, mask=~isequal.S, **opts) + # max(abs(x), abs(y)) + x(**opts) << x.ewise_mult(y, binary.max) + max_x_y = x + # rel_tol * max(abs(x), abs(y)) + max_x_y(**opts) << max_x_y.apply(binary.times, rel_tol) + # max(rel_tol * max(abs(x), abs(y)), abs_tol) + max_x_y(**opts) << max_x_y.apply(binary.max, abs_tol) + + # x - y + y(~isequal.S, replace=True, **opts) << self.ewise_mult(other, binary.minus) + abs_x_y = y + # abs(x - y) + abs_x_y(**opts) << abs_x_y.apply(unary.abs) + + # abs(x - y) <= max(rel_tol * max(abs(x), abs(y)), abs_tol) + isequal(**opts) << abs_x_y.ewise_mult(max_x_y, binary.le) + if isequal.ndim == 1: + return isequal.reduce(monoid.land, allow_empty=False).new(**opts).value + return isequal.reduce_scalar(monoid.land, allow_empty=False).new(**opts).value + + class Vector(BaseType): """Create a new GraphBLAS Sparse Vector. @@ -354,6 +393,8 @@ def isclose(self, other, *, rel_tol=1e-7, abs_tol=0.0, check_dtype=False, **opts return False if self._nvals != other._nvals: return False + if not _supports_udfs: + return _isclose_recipe(self, other, rel_tol, abs_tol, **opts) matches = self.ewise_mult(other, binary.isclose(rel_tol, abs_tol)).new( bool, name="M_isclose", **opts @@ -520,14 +561,15 @@ def build(self, indices, values, *, dup_op=None, clear=False, size=None): if not dup_op_given: if not self.dtype._is_udt: dup_op = binary.plus - else: + elif backend != "suitesparse": dup_op = binary.any - # SS:SuiteSparse-specific: we could use NULL for dup_op - dup_op = get_typed_op(dup_op, self.dtype, kind="binary") - if dup_op.opclass == "Monoid": - dup_op = dup_op.binaryop - else: - self._expect_op(dup_op, "BinaryOp", within="build", argname="dup_op") + # SS:SuiteSparse-specific: we use NULL for dup_op + if dup_op is not None: + dup_op = get_typed_op(dup_op, self.dtype, kind="binary") + if dup_op.opclass == "Monoid": + dup_op = dup_op.binaryop + else: + self._expect_op(dup_op, "BinaryOp", within="build", argname="dup_op") indices = _CArray(indices) values = _CArray(values, self.dtype) @@ -1500,6 +1542,7 @@ def select(self, op, thunk=None): if thunk.dtype._is_udt: dtype_name = "UDT" thunk = _Pointer(thunk) + # NOT COVERED else: dtype_name = thunk.dtype.name cfunc_name = f"GrB_Vector_select_{dtype_name}" @@ -1817,13 +1860,14 @@ def _prep_for_assign(self, resolved_indexes, value, mask, is_submask, replace, o shape = values.shape try: vals = Vector.from_dense(values, dtype=dtype) - except Exception: # pragma: no cover (safety) + except Exception: vals = None else: if dtype.np_type.subdtype is not None: shape = vals.shape if vals is None or shape != (size,): if dtype.np_type.subdtype is not None: + # NOT COVERED extra = ( " (this is assigning to a vector with sub-array dtype " f"({dtype}), so array shape should include dtype shape)" @@ -1943,7 +1987,7 @@ def from_dict(cls, d, dtype=None, *, size=None, name=None): # If we know the dtype, then using `np.fromiter` is much faster dtype = lookup_dtype(dtype) if dtype.np_type.subdtype is not None and np.__version__[:5] in {"1.21.", "1.22."}: - values, dtype = values_to_numpy_buffer(list(d.values()), dtype) + values, dtype = values_to_numpy_buffer(list(d.values()), dtype) # FLAKY COVERAGE else: values = np.fromiter(d.values(), dtype.np_type) if size is None and indices.size == 0: diff --git a/graphblas/dtypes.py b/graphblas/dtypes.py index 22d98b8f1..920610b95 100644 --- a/graphblas/dtypes.py +++ b/graphblas/dtypes.py @@ -1,15 +1,18 @@ import warnings as _warnings -import numba as _numba import numpy as _np from numpy import find_common_type as _find_common_type from numpy import promote_types as _promote_types from . import backend from .core import NULL as _NULL +from .core import _has_numba from .core import ffi as _ffi from .core import lib as _lib +if _has_numba: + import numba as _numba + # Default assumption unless FC32/FC64 are found in lib _supports_complex = hasattr(_lib, "GrB_FC64") or hasattr(_lib, "GxB_FC64") @@ -140,44 +143,126 @@ def register_anonymous(dtype, name=None): # For now, let's use "opaque" unsigned bytes for the c type. if name is None: name = _default_name(dtype) - numba_type = _numba.typeof(dtype).dtype + numba_type = _numba.typeof(dtype).dtype if _has_numba else None rv = DataType(name, gb_obj, None, f"uint8_t[{dtype.itemsize}]", numba_type, dtype) _registry[gb_obj] = rv _registry[dtype] = rv - _registry[numba_type] = rv - _registry[numba_type.name] = rv + if _has_numba: + _registry[numba_type] = rv + _registry[numba_type.name] = rv return rv -BOOL = DataType("BOOL", _lib.GrB_BOOL, "GrB_BOOL", "_Bool", _numba.types.bool_, _np.bool_) -INT8 = DataType("INT8", _lib.GrB_INT8, "GrB_INT8", "int8_t", _numba.types.int8, _np.int8) -UINT8 = DataType("UINT8", _lib.GrB_UINT8, "GrB_UINT8", "uint8_t", _numba.types.uint8, _np.uint8) -INT16 = DataType("INT16", _lib.GrB_INT16, "GrB_INT16", "int16_t", _numba.types.int16, _np.int16) +BOOL = DataType( + "BOOL", + _lib.GrB_BOOL, + "GrB_BOOL", + "_Bool", + _numba.types.bool_ if _has_numba else None, + _np.bool_, +) +INT8 = DataType( + "INT8", _lib.GrB_INT8, "GrB_INT8", "int8_t", _numba.types.int8 if _has_numba else None, _np.int8 +) +UINT8 = DataType( + "UINT8", + _lib.GrB_UINT8, + "GrB_UINT8", + "uint8_t", + _numba.types.uint8 if _has_numba else None, + _np.uint8, +) +INT16 = DataType( + "INT16", + _lib.GrB_INT16, + "GrB_INT16", + "int16_t", + _numba.types.int16 if _has_numba else None, + _np.int16, +) UINT16 = DataType( - "UINT16", _lib.GrB_UINT16, "GrB_UINT16", "uint16_t", _numba.types.uint16, _np.uint16 + "UINT16", + _lib.GrB_UINT16, + "GrB_UINT16", + "uint16_t", + _numba.types.uint16 if _has_numba else None, + _np.uint16, +) +INT32 = DataType( + "INT32", + _lib.GrB_INT32, + "GrB_INT32", + "int32_t", + _numba.types.int32 if _has_numba else None, + _np.int32, ) -INT32 = DataType("INT32", _lib.GrB_INT32, "GrB_INT32", "int32_t", _numba.types.int32, _np.int32) UINT32 = DataType( - "UINT32", _lib.GrB_UINT32, "GrB_UINT32", "uint32_t", _numba.types.uint32, _np.uint32 + "UINT32", + _lib.GrB_UINT32, + "GrB_UINT32", + "uint32_t", + _numba.types.uint32 if _has_numba else None, + _np.uint32, +) +INT64 = DataType( + "INT64", + _lib.GrB_INT64, + "GrB_INT64", + "int64_t", + _numba.types.int64 if _has_numba else None, + _np.int64, ) -INT64 = DataType("INT64", _lib.GrB_INT64, "GrB_INT64", "int64_t", _numba.types.int64, _np.int64) # _Index (like UINT64) is for internal use only and shouldn't be exposed to the user _INDEX = DataType( - "UINT64", _lib.GrB_UINT64, "GrB_Index", "GrB_Index", _numba.types.uint64, _np.uint64 + "UINT64", + _lib.GrB_UINT64, + "GrB_Index", + "GrB_Index", + _numba.types.uint64 if _has_numba else None, + _np.uint64, ) UINT64 = DataType( - "UINT64", _lib.GrB_UINT64, "GrB_UINT64", "uint64_t", _numba.types.uint64, _np.uint64 + "UINT64", + _lib.GrB_UINT64, + "GrB_UINT64", + "uint64_t", + _numba.types.uint64 if _has_numba else None, + _np.uint64, +) +FP32 = DataType( + "FP32", + _lib.GrB_FP32, + "GrB_FP32", + "float", + _numba.types.float32 if _has_numba else None, + _np.float32, +) +FP64 = DataType( + "FP64", + _lib.GrB_FP64, + "GrB_FP64", + "double", + _numba.types.float64 if _has_numba else None, + _np.float64, ) -FP32 = DataType("FP32", _lib.GrB_FP32, "GrB_FP32", "float", _numba.types.float32, _np.float32) -FP64 = DataType("FP64", _lib.GrB_FP64, "GrB_FP64", "double", _numba.types.float64, _np.float64) if _supports_complex and hasattr(_lib, "GxB_FC32"): FC32 = DataType( - "FC32", _lib.GxB_FC32, "GxB_FC32", "float _Complex", _numba.types.complex64, _np.complex64 + "FC32", + _lib.GxB_FC32, + "GxB_FC32", + "float _Complex", + _numba.types.complex64 if _has_numba else None, + _np.complex64, ) if _supports_complex and hasattr(_lib, "GrB_FC32"): # pragma: no cover (unused) FC32 = DataType( - "FC32", _lib.GrB_FC32, "GrB_FC32", "float _Complex", _numba.types.complex64, _np.complex64 + "FC32", + _lib.GrB_FC32, + "GrB_FC32", + "float _Complex", + _numba.types.complex64 if _has_numba else None, + _np.complex64, ) if _supports_complex and hasattr(_lib, "GxB_FC64"): FC64 = DataType( @@ -185,7 +270,7 @@ def register_anonymous(dtype, name=None): _lib.GxB_FC64, "GxB_FC64", "double _Complex", - _numba.types.complex128, + _numba.types.complex128 if _has_numba else None, _np.complex128, ) if _supports_complex and hasattr(_lib, "GrB_FC64"): # pragma: no cover (unused) @@ -194,7 +279,7 @@ def register_anonymous(dtype, name=None): _lib.GrB_FC64, "GrB_FC64", "double _Complex", - _numba.types.complex128, + _numba.types.complex128 if _has_numba else None, _np.complex128, ) @@ -246,8 +331,9 @@ def register_anonymous(dtype, name=None): _registry[dtype.gb_name.lower()] = dtype _registry[dtype.c_type] = dtype _registry[dtype.c_type.upper()] = dtype - _registry[dtype.numba_type] = dtype - _registry[dtype.numba_type.name] = dtype + if _has_numba: + _registry[dtype.numba_type] = dtype + _registry[dtype.numba_type.name] = dtype val = _sample_values[dtype] _registry[val.dtype] = dtype _registry[val.dtype.name] = dtype diff --git a/graphblas/io.py b/graphblas/io.py index bc57c2084..23b9b30b7 100644 --- a/graphblas/io.py +++ b/graphblas/io.py @@ -11,7 +11,7 @@ from .exceptions import GraphblasException as _GraphblasException -def draw(m): # pragma: no cover +def draw(m): # pragma: no cover (deprecated) """Draw a square adjacency Matrix as a graph. Requires `networkx `_ and @@ -455,7 +455,7 @@ def to_awkward(A, format=None): indices, values = A.to_coo() form = RecordForm( contents=[ - NumpyForm(A.dtype.numba_type.name, form_key="node1"), + NumpyForm(A.dtype.np_type.name, form_key="node1"), NumpyForm("int64", form_key="node0"), ], fields=["values", "indices"], @@ -489,7 +489,7 @@ def to_awkward(A, format=None): RecordForm( contents=[ NumpyForm("int64", form_key="node3"), - NumpyForm(A.dtype.numba_type.name, form_key="node4"), + NumpyForm(A.dtype.np_type.name, form_key="node4"), ], fields=["indices", "values"], ), @@ -502,11 +502,11 @@ def to_awkward(A, format=None): @ak.behaviors.mixins.mixin_class(ak.behavior) class _AwkwardDoublyCompressedMatrix: @property - def values(self): + def values(self): # pragma: no branch (???) return self.data.values @property - def indices(self): + def indices(self): # pragma: no branch (???) return self.data.indices form = RecordForm( diff --git a/graphblas/monoid/numpy.py b/graphblas/monoid/numpy.py index 1d687443f..f46d57143 100644 --- a/graphblas/monoid/numpy.py +++ b/graphblas/monoid/numpy.py @@ -5,15 +5,18 @@ https://numba.readthedocs.io/en/stable/reference/numpysupported.html#math-operations """ -import numba as _numba import numpy as _np from .. import _STANDARD_OPERATOR_NAMES from .. import binary as _binary from .. import config as _config from .. import monoid as _monoid +from ..core import _has_numba, _supports_udfs from ..dtypes import _supports_complex +if _has_numba: + import numba as _numba + _delayed = {} _complex_dtypes = {"FC32", "FC64"} _float_dtypes = {"FP32", "FP64"} @@ -86,7 +89,8 @@ # To increase import speed, only call njit when `_config.get("mapnumpy")` is False if ( _config.get("mapnumpy") - or type(_numba.njit(lambda x, y: _np.fmax(x, y))(1, 2)) # pragma: no branch (numba) + or _has_numba + and type(_numba.njit(lambda x, y: _np.fmax(x, y))(1, 2)) # pragma: no branch (numba) is not float ): # Incorrect behavior was introduced in numba 0.56.2 and numpy 1.23 @@ -155,7 +159,12 @@ def __dir__(): - return globals().keys() | _delayed.keys() | _monoid_identities.keys() + if not _supports_udfs and not _config.get("mapnumpy"): + return globals().keys() # FLAKY COVERAGE + attrs = _delayed.keys() | _monoid_identities.keys() + if not _supports_udfs: + attrs &= _numpy_to_graphblas.keys() + return attrs | globals().keys() def __getattr__(name): diff --git a/graphblas/op/__init__.py b/graphblas/op/__init__.py index af05cbef4..1eb2b51d7 100644 --- a/graphblas/op/__init__.py +++ b/graphblas/op/__init__.py @@ -39,10 +39,18 @@ def __getattr__(key): ss = import_module(".ss", __name__) globals()["ss"] = ss return ss + if not _supports_udfs: + from .. import binary, semiring + + if key in binary._udfs or key in semiring._udfs: + raise AttributeError( + f"module {__name__!r} unable to compile UDF for {key!r}; " + "install numba for UDF support" + ) raise AttributeError(f"module {__name__!r} has no attribute {key!r}") -from ..core import operator # noqa: E402 isort:skip +from ..core import operator, _supports_udfs # noqa: E402 isort:skip from . import numpy # noqa: E402 isort:skip del operator diff --git a/graphblas/op/numpy.py b/graphblas/op/numpy.py index 497a6037c..cadba17eb 100644 --- a/graphblas/op/numpy.py +++ b/graphblas/op/numpy.py @@ -1,4 +1,5 @@ from ..binary import numpy as _np_binary +from ..core import _supports_udfs from ..semiring import numpy as _np_semiring from ..unary import numpy as _np_unary @@ -10,7 +11,10 @@ def __dir__(): - return globals().keys() | _delayed.keys() | _op_to_mod.keys() + attrs = _delayed.keys() | _op_to_mod.keys() + if not _supports_udfs: + attrs &= _np_unary.__dir__() | _np_binary.__dir__() | _np_semiring.__dir__() + return attrs | globals().keys() def __getattr__(name): diff --git a/graphblas/semiring/__init__.py b/graphblas/semiring/__init__.py index 904ae192f..538136406 100644 --- a/graphblas/semiring/__init__.py +++ b/graphblas/semiring/__init__.py @@ -1,7 +1,29 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of Semiring instances +from ..core import _supports_udfs + _delayed = {} _deprecated = {} +_udfs = { + # Used by aggregators + "max_absfirst", + "max_abssecond", + "plus_absfirst", + "plus_abssecond", + "plus_rpow", + # floordiv + "any_floordiv", + "max_floordiv", + "min_floordiv", + "plus_floordiv", + "times_floordiv", + # rfloordiv + "any_rfloordiv", + "max_rfloordiv", + "min_rfloordiv", + "plus_rfloordiv", + "times_rfloordiv", +} def __dir__(): @@ -47,6 +69,11 @@ def __getattr__(key): ss = import_module(".ss", __name__) globals()["ss"] = ss return ss + if not _supports_udfs and key in _udfs: + raise AttributeError( + f"module {__name__!r} unable to compile UDF for {key!r}; " + "install numba for UDF support" + ) raise AttributeError(f"module {__name__!r} has no attribute {key!r}") diff --git a/graphblas/semiring/numpy.py b/graphblas/semiring/numpy.py index e47ac0336..3a59090cc 100644 --- a/graphblas/semiring/numpy.py +++ b/graphblas/semiring/numpy.py @@ -12,6 +12,7 @@ from .. import config as _config from .. import monoid as _monoid from ..binary.numpy import _binary_names +from ..core import _supports_udfs from ..monoid.numpy import _fmin_is_float, _monoid_identities _delayed = {} @@ -132,7 +133,17 @@ def __dir__(): - return globals().keys() | _delayed.keys() | _semiring_names + if not _supports_udfs and not _config.get("mapnumpy"): + return globals().keys() # FLAKY COVERAGE + attrs = _delayed.keys() | _semiring_names + if not _supports_udfs: + attrs &= { + f"{monoid_name}_{binary_name}" + for monoid_name, binary_name in _itertools.product( + dir(_monoid.numpy), dir(_binary.numpy) + ) + } + return attrs | globals().keys() def __getattr__(name): diff --git a/graphblas/tests/conftest.py b/graphblas/tests/conftest.py index 24aba085f..a4df5d336 100644 --- a/graphblas/tests/conftest.py +++ b/graphblas/tests/conftest.py @@ -1,16 +1,20 @@ import atexit import functools import itertools +import platform from pathlib import Path import numpy as np import pytest import graphblas as gb +from graphblas.core import _supports_udfs as supports_udfs orig_binaryops = set() orig_semirings = set() +pypy = platform.python_implementation() == "PyPy" + def pytest_configure(config): rng = np.random.default_rng() @@ -48,7 +52,7 @@ def pytest_configure(config): rec.start() def save_records(): - with Path("record.txt").open("w") as f: # pragma: no cover + with Path("record.txt").open("w") as f: # pragma: no cover (???) f.write("\n".join(rec.data)) # I'm sure there's a `pytest` way to do this... @@ -116,3 +120,8 @@ def inner(*args, **kwargs): def compute(x): return x + + +def shouldhave(module, opname): + """Whether an "operator" module should have the given operator.""" + return supports_udfs or hasattr(module, opname) diff --git a/graphblas/tests/test_core.py b/graphblas/tests/test_core.py index 71d0bd8a3..ae2051145 100644 --- a/graphblas/tests/test_core.py +++ b/graphblas/tests/test_core.py @@ -80,7 +80,7 @@ def test_packages(): pkgs.append("graphblas") pkgs.sort() pyproject = path.parent / "pyproject.toml" - if not pyproject.exists(): + if not pyproject.exists(): # pragma: no cover (safety) pytest.skip("Did not find pyproject.toml") with pyproject.open("rb") as f: pkgs2 = sorted(tomli.load(f)["tool"]["setuptools"]["packages"]) diff --git a/graphblas/tests/test_dtype.py b/graphblas/tests/test_dtype.py index 64e6d69ab..66c19cce5 100644 --- a/graphblas/tests/test_dtype.py +++ b/graphblas/tests/test_dtype.py @@ -252,7 +252,4 @@ def test_has_complex(): import suitesparse_graphblas as ssgb from packaging.version import parse - if parse(ssgb.__version__) < parse("7.4.3.1"): - assert not dtypes._supports_complex - else: - assert dtypes._supports_complex + assert dtypes._supports_complex == (parse(ssgb.__version__) >= parse("7.4.3.1")) diff --git a/graphblas/tests/test_formatting.py b/graphblas/tests/test_formatting.py index 3094aea91..faadc983b 100644 --- a/graphblas/tests/test_formatting.py +++ b/graphblas/tests/test_formatting.py @@ -40,9 +40,8 @@ def _printer(text, name, repr_name, indent): # line = f"f'{{CSS_STYLE}}'" in_style = False is_style = True - else: # pragma: no cover (???) - # This definitely gets covered, but why is it not picked up? - continue + else: + continue # FLAKY COVERAGE if repr_name == "repr_html" and line.startswith("
" + f"{CSS_STYLE}" + '
vA
\n' + '\n' + " \n" + ' \n' + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "
gb.Vector
nvals
size
dtype
format
12INT64bitmap (iso)
\n" + "
\n" + "
\n" + "\n" + '\n' + " \n" + ' \n' + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "
01
2
\n" + "
" + ) diff --git a/graphblas/tests/test_infix.py b/graphblas/tests/test_infix.py index 14af6108c..72e1c8a42 100644 --- a/graphblas/tests/test_infix.py +++ b/graphblas/tests/test_infix.py @@ -360,3 +360,10 @@ def test_infix_expr_value_types(): assert expr._expr is not None assert expr._value is None assert type(expr.new()) is Matrix + assert type(expr._get_value()) is Matrix + assert expr._expr is not None + assert expr._value is not None + assert expr._expr._value is not None + expr._value = None + assert expr._value is None + assert expr._expr._value is None diff --git a/graphblas/tests/test_io.py b/graphblas/tests/test_io.py index ada092025..24df55e9d 100644 --- a/graphblas/tests/test_io.py +++ b/graphblas/tests/test_io.py @@ -267,9 +267,13 @@ def test_mmread_mmwrite(engine): # fast_matrix_market v1.4.5 raises ValueError instead of OverflowError M = gb.io.mmread(mm_in, engine) else: - if example == "_empty_lines_example" and engine in {"fmm", "auto"} and fmm is not None: - # TODO MAINT: is this a bug in fast_matrix_market, or does scipy.io.mmread - # read an invalid file? `fast_matrix_market` v1.4.5 does not handle this. + if ( + example == "_empty_lines_example" + and engine in {"fmm", "auto"} + and fmm is not None + and fmm.__version__ in {"1.4.5"} + ): + # `fast_matrix_market` __version__ v1.4.5 does not handle this, but v1.5.0 does continue M = gb.io.mmread(mm_in, engine) if not M.isequal(expected): # pragma: no cover (debug) diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index 1d42035a3..26017f364 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -11,6 +11,7 @@ import graphblas as gb from graphblas import agg, backend, binary, dtypes, indexunary, monoid, select, semiring, unary +from graphblas.core import _supports_udfs as supports_udfs from graphblas.core import lib from graphblas.exceptions import ( DimensionMismatch, @@ -23,7 +24,7 @@ OutputNotEmpty, ) -from .conftest import autocompute, compute +from .conftest import autocompute, compute, pypy, shouldhave from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) @@ -1230,6 +1231,8 @@ def test_apply_indexunary(A): assert w4.isequal(A3) with pytest.raises(TypeError, match="left"): A.apply(select.valueeq, left=s3) + assert pickle.loads(pickle.dumps(indexunary.tril)) is indexunary.tril + assert pickle.loads(pickle.dumps(indexunary.tril[int])) is indexunary.tril[int] def test_select(A): @@ -1259,6 +1262,16 @@ def test_select(A): with pytest.raises(TypeError, match="thunk"): A.select(select.valueeq, object()) + A3rows = Matrix.from_coo([0, 0, 1, 1, 2], [1, 3, 4, 6, 5], [2, 3, 8, 4, 1], nrows=7, ncols=7) + w8 = select.rowle(A, 2).new() + w9 = A.select("row<=", 2).new() + w10 = select.row(A < 3).new() + assert w8.isequal(A3rows) + assert w9.isequal(A3rows) + assert w10.isequal(A3rows) + assert pickle.loads(pickle.dumps(select.tril)) is select.tril + assert pickle.loads(pickle.dumps(select.tril[bool])) is select.tril[bool] + @autocompute def test_select_bools_and_masks(A): @@ -1283,16 +1296,27 @@ def test_select_bools_and_masks(A): A.select(A[0, :].new().S) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_indexunary_udf(A): def threex_minusthunk(x, row, col, thunk): # pragma: no cover (numba) return 3 * x - thunk - indexunary.register_new("threex_minusthunk", threex_minusthunk) + assert indexunary.register_new("threex_minusthunk", threex_minusthunk) is not None assert hasattr(indexunary, "threex_minusthunk") assert not hasattr(select, "threex_minusthunk") with pytest.raises(ValueError, match="SelectOp must have BOOL return type"): select.register_anonymous(threex_minusthunk) + with pytest.raises(ValueError, match="SelectOp must have BOOL return type"): + select.register_new("bad_select", threex_minusthunk) + assert not hasattr(indexunary, "bad_select") + assert not hasattr(select, "bad_select") + assert select.register_new("bad_select", threex_minusthunk, lazy=True) is None + with pytest.raises(ValueError, match="SelectOp must have BOOL return type"): + select.bad_select + assert not hasattr(select, "bad_select") + assert hasattr(indexunary, "bad_select") # Keep it + expected = Matrix.from_coo( [3, 0, 3, 5, 6, 0, 6, 1, 6, 2, 4, 1], [0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6], @@ -1308,6 +1332,8 @@ def iii(x, row, col, thunk): # pragma: no cover (numba) select.register_new("iii", iii) assert hasattr(indexunary, "iii") assert hasattr(select, "iii") + assert indexunary.iii[int].orig_func is select.iii[int].orig_func is select.iii.orig_func + assert indexunary.iii[int]._numba_func is select.iii[int]._numba_func is select.iii._numba_func iii_apply = indexunary.register_anonymous(iii) expected = Matrix.from_coo( [3, 0, 3, 5, 6, 0, 6, 1, 6, 2, 4, 1], @@ -1353,15 +1379,17 @@ def test_reduce_agg(A): expected = unary.sqrt[float](squared).new() w5 = A.reduce_rowwise(agg.hypot).new() assert w5.isclose(expected) - w6 = A.reduce_rowwise(monoid.numpy.hypot[float]).new() - assert w6.isclose(expected) + if shouldhave(monoid.numpy, "hypot"): + w6 = A.reduce_rowwise(monoid.numpy.hypot[float]).new() + assert w6.isclose(expected) w7 = Vector(w5.dtype, size=w5.size) w7 << A.reduce_rowwise(agg.hypot) assert w7.isclose(expected) w8 = A.reduce_rowwise(agg.logaddexp).new() - expected = A.reduce_rowwise(monoid.numpy.logaddexp[float]).new() - assert w8.isclose(w8) + if shouldhave(monoid.numpy, "logaddexp"): + expected = A.reduce_rowwise(monoid.numpy.logaddexp[float]).new() + assert w8.isclose(w8) result = Vector.from_coo([0, 1, 2, 3, 4, 5, 6], [3, 2, 9, 10, 11, 8, 4]) w9 = A.reduce_columnwise(agg.sum).new() @@ -1598,6 +1626,7 @@ def test_reduce_agg_empty(): assert compute(s.value) is None +@pytest.mark.skipif("not supports_udfs") def test_reduce_row_udf(A): result = Vector.from_coo([0, 1, 2, 3, 4, 5, 6], [5, 12, 1, 6, 7, 1, 15]) @@ -2007,6 +2036,12 @@ def test_ss_import_export(A, do_iso, methods): B4 = Matrix.ss.import_any(**d) assert B4.isequal(A) assert B4.ss.is_iso is do_iso + if do_iso: + d["values"] = 1 + d["is_iso"] = False + B4b = Matrix.ss.import_any(**d) + assert B4b.isequal(A) + assert B4b.ss.is_iso is True else: A4.ss.pack_any(**d) assert A4.isequal(A) @@ -2262,6 +2297,11 @@ def test_ss_import_on_view(): A = Matrix.from_coo([0, 0, 1, 1], [0, 1, 0, 1], [1, 2, 3, 4]) B = Matrix.ss.import_any(nrows=2, ncols=2, values=np.array([1, 2, 3, 4, 99, 99, 99])[:4]) assert A.isequal(B) + values = np.arange(16).reshape(4, 4)[::2, ::2] + bitmap = np.ones((4, 4), dtype=bool)[::2, ::2] + C = Matrix.ss.import_any(values=values, bitmap=bitmap) + D = Matrix.ss.import_any(values=values.copy(), bitmap=bitmap.copy()) + assert C.isequal(D) @pytest.mark.skipif("not suitesparse") @@ -2902,18 +2942,19 @@ def test_expr_is_like_matrix(A): "resize", "update", } - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Matrix. You may need to " "add an entry to `matrix` or `matrix_vector` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " "methods, then you may need to run `python -m graphblas.core.infixmethods`." ) - assert attrs - infix_attrs == expected + assert attrs - infix_attrs - ignore == expected # TransposedMatrix is used differently than other expressions, # so maybe it shouldn't support everything. if suitesparse: expected.add("ss") - assert attrs - transposed_attrs == (expected | {"_as_vector", "S", "V"}) - { + assert attrs - transposed_attrs - ignore == (expected | {"_as_vector", "S", "V"}) - { "_prep_for_extract", "_extract_element", } @@ -2965,7 +3006,8 @@ def test_index_expr_is_like_matrix(A): "from_scalar", "resize", } - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Matrix. You may need to " "add an entry to `matrix` or `matrix_vector` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " @@ -3110,10 +3152,12 @@ def test_infix_sugar(A): assert binary.times(2, A).isequal(2 * A) assert binary.truediv(A, 2).isequal(A / 2) assert binary.truediv(5, A).isequal(5 / A) - assert binary.floordiv(A, 2).isequal(A // 2) - assert binary.floordiv(5, A).isequal(5 // A) - assert binary.numpy.mod(A, 2).isequal(A % 2) - assert binary.numpy.mod(5, A).isequal(5 % A) + if shouldhave(binary, "floordiv"): + assert binary.floordiv(A, 2).isequal(A // 2) + assert binary.floordiv(5, A).isequal(5 // A) + if shouldhave(binary.numpy, "mod"): + assert binary.numpy.mod(A, 2).isequal(A % 2) + assert binary.numpy.mod(5, A).isequal(5 % A) assert binary.pow(A, 2).isequal(A**2) assert binary.pow(2, A).isequal(2**A) assert binary.pow(A, 2).isequal(pow(A, 2)) @@ -3140,26 +3184,27 @@ def test_infix_sugar(A): assert binary.ge(A, 4).isequal(A >= 4) assert binary.eq(A, 4).isequal(A == 4) assert binary.ne(A, 4).isequal(A != 4) - x, y = divmod(A, 3) - assert binary.floordiv(A, 3).isequal(x) - assert binary.numpy.mod(A, 3).isequal(y) - assert binary.fmod(A, 3).isequal(y) - assert A.isequal(binary.plus((3 * x) & y)) - x, y = divmod(-A, 3) - assert binary.floordiv(-A, 3).isequal(x) - assert binary.numpy.mod(-A, 3).isequal(y) - # assert binary.fmod(-A, 3).isequal(y) # The reason we use numpy.mod - assert (-A).isequal(binary.plus((3 * x) & y)) - x, y = divmod(3, A) - assert binary.floordiv(3, A).isequal(x) - assert binary.numpy.mod(3, A).isequal(y) - assert binary.fmod(3, A).isequal(y) - assert binary.plus(binary.times(A & x) & y).isequal(3 * unary.one(A)) - x, y = divmod(-3, A) - assert binary.floordiv(-3, A).isequal(x) - assert binary.numpy.mod(-3, A).isequal(y) - # assert binary.fmod(-3, A).isequal(y) # The reason we use numpy.mod - assert binary.plus(binary.times(A & x) & y).isequal(-3 * unary.one(A)) + if shouldhave(binary, "floordiv") and shouldhave(binary.numpy, "mod"): + x, y = divmod(A, 3) + assert binary.floordiv(A, 3).isequal(x) + assert binary.numpy.mod(A, 3).isequal(y) + assert binary.fmod(A, 3).isequal(y) + assert A.isequal(binary.plus((3 * x) & y)) + x, y = divmod(-A, 3) + assert binary.floordiv(-A, 3).isequal(x) + assert binary.numpy.mod(-A, 3).isequal(y) + # assert binary.fmod(-A, 3).isequal(y) # The reason we use numpy.mod + assert (-A).isequal(binary.plus((3 * x) & y)) + x, y = divmod(3, A) + assert binary.floordiv(3, A).isequal(x) + assert binary.numpy.mod(3, A).isequal(y) + assert binary.fmod(3, A).isequal(y) + assert binary.plus(binary.times(A & x) & y).isequal(3 * unary.one(A)) + x, y = divmod(-3, A) + assert binary.floordiv(-3, A).isequal(x) + assert binary.numpy.mod(-3, A).isequal(y) + # assert binary.fmod(-3, A).isequal(y) # The reason we use numpy.mod + assert binary.plus(binary.times(A & x) & y).isequal(-3 * unary.one(A)) assert binary.eq(A & A).isequal(A == A) assert binary.ne(A.T & A.T).isequal(A.T != A.T) @@ -3182,14 +3227,16 @@ def test_infix_sugar(A): B /= 2 assert type(B) is Matrix assert binary.truediv(A, 2).isequal(B) - B = A.dup() - B //= 2 - assert type(B) is Matrix - assert binary.floordiv(A, 2).isequal(B) - B = A.dup() - B %= 2 - assert type(B) is Matrix - assert binary.numpy.mod(A, 2).isequal(B) + if shouldhave(binary, "floordiv"): + B = A.dup() + B //= 2 + assert type(B) is Matrix + assert binary.floordiv(A, 2).isequal(B) + if shouldhave(binary.numpy, "mod"): + B = A.dup() + B %= 2 + assert type(B) is Matrix + assert binary.numpy.mod(A, 2).isequal(B) B = A.dup() B **= 2 assert type(B) is Matrix @@ -3520,7 +3567,7 @@ def test_ndim(A): def test_sizeof(A): - if suitesparse: + if suitesparse and not pypy: assert sys.getsizeof(A) > A.nvals * 16 else: with pytest.raises(TypeError): @@ -3607,6 +3654,7 @@ def test_ss_iteration(A): assert next(A.ss.iteritems()) is not None +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_udt(): record_dtype = np.dtype([("x", np.bool_), ("y", np.float64)], align=True) @@ -3917,7 +3965,7 @@ def test_ss_config(A): def test_to_csr_from_csc(A): - assert Matrix.from_csr(*A.to_csr(dtype=int)).isequal(A, check_dtype=True) + assert Matrix.from_csr(*A.to_csr(sort=False, dtype=int)).isequal(A, check_dtype=True) assert Matrix.from_csr(*A.T.to_csc()).isequal(A, check_dtype=True) assert Matrix.from_csc(*A.to_csc()).isequal(A) assert Matrix.from_csc(*A.T.to_csr()).isequal(A) @@ -4126,7 +4174,11 @@ def test_from_scalar(): A = Matrix.from_scalar(1, dtype="INT64[2]", nrows=3, ncols=4) B = Matrix("INT64[2]", nrows=3, ncols=4) B << [1, 1] - assert A.isequal(B, check_dtype=True) + if supports_udfs: + assert A.isequal(B, check_dtype=True) + else: + with pytest.raises(KeyError, match="eq does not work with"): + assert A.isequal(B, check_dtype=True) def test_to_dense_from_dense(): @@ -4252,7 +4304,7 @@ def test_ss_descriptors(A): (A @ A).new(nthreads=4, Nthreads=5) with pytest.raises(ValueError, match="escriptor"): A[0, 0].new(bad_opt=True) - A[0, 0].new(nthreads=4) # ignored, but okay + A[0, 0].new(nthreads=4, sort=None) # ignored, but okay with pytest.raises(ValueError, match="escriptor"): A.__setitem__((0, 0), 1, bad_opt=True) A.__setitem__((0, 0), 1, nthreads=4) # ignored, but okay @@ -4286,6 +4338,7 @@ def test_wait_chains(A): assert result == 47 +@pytest.mark.skipif("not supports_udfs") def test_subarray_dtypes(): a = np.arange(3 * 4, dtype=np.int64).reshape(3, 4) A = Matrix.from_coo([1, 3, 5], [0, 1, 3], a) diff --git a/graphblas/tests/test_numpyops.py b/graphblas/tests/test_numpyops.py index 5b7e797f3..25c52d7fd 100644 --- a/graphblas/tests/test_numpyops.py +++ b/graphblas/tests/test_numpyops.py @@ -11,22 +11,25 @@ import graphblas.monoid.numpy as npmonoid import graphblas.semiring.numpy as npsemiring import graphblas.unary.numpy as npunary -from graphblas import Vector, backend +from graphblas import Vector, backend, config +from graphblas.core import _supports_udfs as supports_udfs from graphblas.dtypes import _supports_complex -from .conftest import compute +from .conftest import compute, shouldhave is_win = sys.platform.startswith("win") suitesparse = backend == "suitesparse" def test_numpyops_dir(): - assert "exp2" in dir(npunary) - assert "logical_and" in dir(npbinary) - assert "logaddexp" in dir(npmonoid) - assert "add_add" in dir(npsemiring) + udf_or_mapped = supports_udfs or config["mapnumpy"] + assert ("exp2" in dir(npunary)) == udf_or_mapped + assert ("logical_and" in dir(npbinary)) == udf_or_mapped + assert ("logaddexp" in dir(npmonoid)) == supports_udfs + assert ("add_add" in dir(npsemiring)) == udf_or_mapped +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_bool_doesnt_get_too_large(): a = Vector.from_coo([0, 1, 2, 3], [True, False, True, False]) @@ -70,9 +73,12 @@ def test_npunary(): # due to limitation of MSVC with complex blocklist["FC64"].update({"arcsin", "arcsinh"}) blocklist["FC32"] = {"arcsin", "arcsinh"} - isclose = gb.binary.isclose(1e-6, 0) + if shouldhave(gb.binary, "isclose"): + isclose = gb.binary.isclose(1e-6, 0) + else: + isclose = None for gb_input, np_input in data: - for unary_name in sorted(npunary._unary_names): + for unary_name in sorted(npunary._unary_names & npunary.__dir__()): op = getattr(npunary, unary_name) if gb_input.dtype not in op.types or unary_name in blocklist.get( gb_input.dtype.name, () @@ -99,6 +105,8 @@ def test_npunary(): list(range(np_input.size)), list(np_result), dtype=gb_result.dtype ) assert gb_result.nvals == np_result.size + if compare_op is None: + continue # FLAKY COVERAGE match = gb_result.ewise_mult(np_result, compare_op).new() if gb_result.dtype.name.startswith("F"): match(accum=gb.binary.lor) << gb_result.apply(npunary.isnan) @@ -149,9 +157,24 @@ def test_npbinary(): "FP64": {"floor_divide"}, # numba/numpy difference for 1.0 / 0.0 "BOOL": {"gcd", "lcm", "subtract"}, # not supported by numpy } - isclose = gb.binary.isclose(1e-7, 0) + if shouldhave(gb.binary, "isclose"): + isclose = gb.binary.isclose(1e-7, 0) + else: + isclose = None + if shouldhave(npbinary, "equal"): + equal = npbinary.equal + else: + equal = gb.binary.eq + if shouldhave(npbinary, "isnan"): + isnan = npunary.isnan + else: + isnan = gb.unary.isnan + if shouldhave(npbinary, "isinf"): + isinf = npunary.isinf + else: + isinf = gb.unary.isinf for (gb_left, gb_right), (np_left, np_right) in data: - for binary_name in sorted(npbinary._binary_names): + for binary_name in sorted(npbinary._binary_names & npbinary.__dir__()): op = getattr(npbinary, binary_name) if gb_left.dtype not in op.types or binary_name in blocklist.get( gb_left.dtype.name, () @@ -171,7 +194,7 @@ def test_npbinary(): if binary_name in {"arctan2"}: compare_op = isclose else: - compare_op = npbinary.equal + compare_op = equal except Exception: # pragma: no cover (debug) print(f"Error computing numpy result for {binary_name}") print(f"dtypes: ({gb_left.dtype}, {gb_right.dtype}) -> {gb_result.dtype}") @@ -179,12 +202,14 @@ def test_npbinary(): np_result = Vector.from_coo(np.arange(np_left.size), np_result, dtype=gb_result.dtype) assert gb_result.nvals == np_result.size + if compare_op is None: + continue # FLAKY COVERAGE match = gb_result.ewise_mult(np_result, compare_op).new() if gb_result.dtype.name.startswith("F"): - match(accum=gb.binary.lor) << gb_result.apply(npunary.isnan) + match(accum=gb.binary.lor) << gb_result.apply(isnan) if gb_result.dtype.name.startswith("FC"): # Divide by 0j sometimes result in different behavior, such as `nan` or `(inf+0j)` - match(accum=gb.binary.lor) << gb_result.apply(npunary.isinf) + match(accum=gb.binary.lor) << gb_result.apply(isinf) compare = match.reduce(gb.monoid.land).new() if not compare: # pragma: no cover (debug) print(compare_op) @@ -223,7 +248,7 @@ def test_npmonoid(): ], ] # Complex monoids not working yet (they segfault upon creation in gb.core.operators) - if _supports_complex: # pragma: no branch + if _supports_complex: data.append( [ [ @@ -241,13 +266,13 @@ def test_npmonoid(): "BOOL": {"add"}, } for (gb_left, gb_right), (np_left, np_right) in data: - for binary_name in sorted(npmonoid._monoid_identities): + for binary_name in sorted(npmonoid._monoid_identities.keys() & npmonoid.__dir__()): op = getattr(npmonoid, binary_name) assert len(op.types) > 0, op.name if gb_left.dtype not in op.types or binary_name in blocklist.get( gb_left.dtype.name, () - ): # pragma: no cover (flaky) - continue + ): + continue # FLAKY COVERAGE with np.errstate(divide="ignore", over="ignore", under="ignore", invalid="ignore"): gb_result = gb_left.ewise_mult(gb_right, op).new() np_result = getattr(np, binary_name)(np_left, np_right) @@ -279,7 +304,8 @@ def test_npmonoid(): @pytest.mark.slow def test_npsemiring(): for monoid_name, binary_name in itertools.product( - sorted(npmonoid._monoid_identities), sorted(npbinary._binary_names) + sorted(npmonoid._monoid_identities.keys() & npmonoid.__dir__()), + sorted(npbinary._binary_names & npbinary.__dir__()), ): monoid = getattr(npmonoid, monoid_name) binary = getattr(npbinary, binary_name) diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index 3a80dbe52..c9a176afd 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -4,7 +4,20 @@ import pytest import graphblas as gb -from graphblas import agg, backend, binary, dtypes, indexunary, monoid, op, select, semiring, unary +from graphblas import ( + agg, + backend, + binary, + config, + dtypes, + indexunary, + monoid, + op, + select, + semiring, + unary, +) +from graphblas.core import _supports_udfs as supports_udfs from graphblas.core import lib, operator from graphblas.core.operator import BinaryOp, IndexUnaryOp, Monoid, Semiring, UnaryOp, get_semiring from graphblas.dtypes import ( @@ -22,6 +35,8 @@ ) from graphblas.exceptions import DomainMismatch, UdfParseError +from .conftest import shouldhave + if dtypes._supports_complex: from graphblas.dtypes import FC32, FC64 @@ -142,6 +157,36 @@ def test_get_typed_op(): operator.get_typed_op(binary.plus, dtypes.INT64, "bad dtype") +@pytest.mark.skipif("supports_udfs") +def test_udf_mentions_numba(): + with pytest.raises(AttributeError, match="install numba"): + binary.rfloordiv + assert "rfloordiv" not in dir(binary) + with pytest.raises(AttributeError, match="install numba"): + semiring.any_rfloordiv + assert "any_rfloordiv" not in dir(semiring) + with pytest.raises(AttributeError, match="install numba"): + op.absfirst + assert "absfirst" not in dir(op) + with pytest.raises(AttributeError, match="install numba"): + op.plus_rpow + assert "plus_rpow" not in dir(op) + with pytest.raises(AttributeError, match="install numba"): + binary.numpy.gcd + assert "gcd" not in dir(binary.numpy) + assert "gcd" not in dir(op.numpy) + + +@pytest.mark.skipif("supports_udfs") +def test_unaryop_udf_no_support(): + def plus_one(x): # pragma: no cover (numba) + return x + 1 + + with pytest.raises(RuntimeError, match="UnaryOp.register_new.* unavailable"): + unary.register_new("plus_one", plus_one) + + +@pytest.mark.skipif("not supports_udfs") def test_unaryop_udf(): def plus_one(x): return x + 1 # pragma: no cover (numba) @@ -150,6 +195,7 @@ def plus_one(x): assert hasattr(unary, "plus_one") assert unary.plus_one.orig_func is plus_one assert unary.plus_one[int].orig_func is plus_one + assert unary.plus_one[int]._numba_func(1) == 2 comp_set = { INT8, INT16, @@ -182,6 +228,7 @@ def plus_one(x): UnaryOp.register_new("bad", lambda x: v) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_unaryop_parameterized(): def plus_x(x=0): @@ -207,6 +254,7 @@ def inner(val): assert r10.isequal(v11, check_dtype=True) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_binaryop_parameterized(): def plus_plus_x(x=0): @@ -268,6 +316,7 @@ def my_add(x, y): assert op.name == "my_add" +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_monoid_parameterized(): def plus_plus_x(x=0): @@ -363,6 +412,7 @@ def bad_identity(x=0): assert monoid.is_idempotent +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_semiring_parameterized(): def plus_plus_x(x=0): @@ -490,6 +540,7 @@ def inner(y): assert B.isequal(A.kronecker(A, binary.plus).new()) +@pytest.mark.skipif("not supports_udfs") def test_unaryop_udf_bool_result(): # numba has trouble compiling this, but we have a work-around def is_positive(x): @@ -516,12 +567,14 @@ def is_positive(x): assert w.isequal(result) +@pytest.mark.skipif("not supports_udfs") def test_binaryop_udf(): def times_minus_sum(x, y): return x * y - (x + y) # pragma: no cover (numba) BinaryOp.register_new("bin_test_func", times_minus_sum) assert hasattr(binary, "bin_test_func") + assert binary.bin_test_func[int].orig_func is times_minus_sum comp_set = { BOOL, # goes to INT64 INT8, @@ -545,6 +598,7 @@ def times_minus_sum(x, y): assert w.isequal(result) +@pytest.mark.skipif("not supports_udfs") def test_monoid_udf(): def plus_plus_one(x, y): return x + y + 1 # pragma: no cover (numba) @@ -579,6 +633,7 @@ def plus_plus_one(x, y): Monoid.register_anonymous(binary.plus_plus_one, {"BOOL": -1}) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_semiring_udf(): def plus_plus_two(x, y): @@ -608,10 +663,12 @@ def test_binary_updates(): vec4 = Vector.from_coo([0], [-3], dtype=dtypes.INT64) result2 = vec4.ewise_mult(vec2, binary.cdiv).new() assert result2.isequal(Vector.from_coo([0], [-1], dtype=dtypes.INT64), check_dtype=True) - result3 = vec4.ewise_mult(vec2, binary.floordiv).new() - assert result3.isequal(Vector.from_coo([0], [-2], dtype=dtypes.INT64), check_dtype=True) + if shouldhave(binary, "floordiv"): + result3 = vec4.ewise_mult(vec2, binary.floordiv).new() + assert result3.isequal(Vector.from_coo([0], [-2], dtype=dtypes.INT64), check_dtype=True) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_nested_names(): def plus_three(x): @@ -671,12 +728,17 @@ def test_op_namespace(): assert op.plus is binary.plus assert op.plus_times is semiring.plus_times - assert op.numpy.fabs is unary.numpy.fabs - assert op.numpy.subtract is binary.numpy.subtract - assert op.numpy.add is binary.numpy.add - assert op.numpy.add_add is semiring.numpy.add_add + if shouldhave(unary.numpy, "fabs"): + assert op.numpy.fabs is unary.numpy.fabs + if shouldhave(binary.numpy, "subtract"): + assert op.numpy.subtract is binary.numpy.subtract + if shouldhave(binary.numpy, "add"): + assert op.numpy.add is binary.numpy.add + if shouldhave(semiring.numpy, "add_add"): + assert op.numpy.add_add is semiring.numpy.add_add assert len(dir(op)) > 300 - assert len(dir(op.numpy)) > 500 + if supports_udfs: + assert len(dir(op.numpy)) > 500 with pytest.raises( AttributeError, match="module 'graphblas.op.numpy' has no attribute 'bad_attr'" @@ -740,10 +802,18 @@ def test_op_namespace(): @pytest.mark.slow def test_binaryop_attributes_numpy(): # Some coverage from this test depends on order of tests - assert binary.numpy.add[int].monoid is monoid.numpy.add[int] - assert binary.numpy.subtract[int].monoid is None - assert binary.numpy.add.monoid is monoid.numpy.add - assert binary.numpy.subtract.monoid is None + if shouldhave(monoid.numpy, "add"): + assert binary.numpy.add[int].monoid is monoid.numpy.add[int] + assert binary.numpy.add.monoid is monoid.numpy.add + if shouldhave(binary.numpy, "subtract"): + assert binary.numpy.subtract[int].monoid is None + assert binary.numpy.subtract.monoid is None + + +@pytest.mark.skipif("not supports_udfs") +@pytest.mark.slow +def test_binaryop_monoid_numpy(): + assert gb.binary.numpy.minimum[int].monoid is gb.monoid.numpy.minimum[int] @pytest.mark.slow @@ -756,18 +826,21 @@ def test_binaryop_attributes(): def plus(x, y): return x + y # pragma: no cover (numba) - op = BinaryOp.register_anonymous(plus, name="plus") - assert op.monoid is None - assert op[int].monoid is None + if supports_udfs: + op = BinaryOp.register_anonymous(plus, name="plus") + assert op.monoid is None + assert op[int].monoid is None + assert op[int].parent is op assert binary.plus[int].parent is binary.plus - assert binary.numpy.add[int].parent is binary.numpy.add - assert op[int].parent is op + if shouldhave(binary.numpy, "add"): + assert binary.numpy.add[int].parent is binary.numpy.add # bad type assert binary.plus[bool].monoid is None - assert binary.numpy.equal[int].monoid is None - assert binary.numpy.equal[bool].monoid is monoid.numpy.equal[bool] # sanity + if shouldhave(binary.numpy, "equal"): + assert binary.numpy.equal[int].monoid is None + assert binary.numpy.equal[bool].monoid is monoid.numpy.equal[bool] # sanity for attr, val in vars(binary).items(): if not isinstance(val, BinaryOp): @@ -790,22 +863,25 @@ def test_monoid_attributes(): assert monoid.plus.binaryop is binary.plus assert monoid.plus.identities == {typ: 0 for typ in monoid.plus.types} - assert monoid.numpy.add[int].binaryop is binary.numpy.add[int] - assert monoid.numpy.add[int].identity == 0 - assert monoid.numpy.add.binaryop is binary.numpy.add - assert monoid.numpy.add.identities == {typ: 0 for typ in monoid.numpy.add.types} + if shouldhave(monoid.numpy, "add"): + assert monoid.numpy.add[int].binaryop is binary.numpy.add[int] + assert monoid.numpy.add[int].identity == 0 + assert monoid.numpy.add.binaryop is binary.numpy.add + assert monoid.numpy.add.identities == {typ: 0 for typ in monoid.numpy.add.types} def plus(x, y): # pragma: no cover (numba) return x + y - binop = BinaryOp.register_anonymous(plus, name="plus") - op = Monoid.register_anonymous(binop, 0, name="plus") - assert op.binaryop is binop - assert op[int].binaryop is binop[int] + if supports_udfs: + binop = BinaryOp.register_anonymous(plus, name="plus") + op = Monoid.register_anonymous(binop, 0, name="plus") + assert op.binaryop is binop + assert op[int].binaryop is binop[int] + assert op[int].parent is op assert monoid.plus[int].parent is monoid.plus - assert monoid.numpy.add[int].parent is monoid.numpy.add - assert op[int].parent is op + if shouldhave(monoid.numpy, "add"): + assert monoid.numpy.add[int].parent is monoid.numpy.add for attr, val in vars(monoid).items(): if not isinstance(val, Monoid): @@ -826,25 +902,27 @@ def test_semiring_attributes(): assert semiring.min_plus.monoid is monoid.min assert semiring.min_plus.binaryop is binary.plus - assert semiring.numpy.add_subtract[int].monoid is monoid.numpy.add[int] - assert semiring.numpy.add_subtract[int].binaryop is binary.numpy.subtract[int] - assert semiring.numpy.add_subtract.monoid is monoid.numpy.add - assert semiring.numpy.add_subtract.binaryop is binary.numpy.subtract + if shouldhave(semiring.numpy, "add_subtract"): + assert semiring.numpy.add_subtract[int].monoid is monoid.numpy.add[int] + assert semiring.numpy.add_subtract[int].binaryop is binary.numpy.subtract[int] + assert semiring.numpy.add_subtract.monoid is monoid.numpy.add + assert semiring.numpy.add_subtract.binaryop is binary.numpy.subtract + assert semiring.numpy.add_subtract[int].parent is semiring.numpy.add_subtract def plus(x, y): return x + y # pragma: no cover (numba) - binop = BinaryOp.register_anonymous(plus, name="plus") - mymonoid = Monoid.register_anonymous(binop, 0, name="plus") - op = Semiring.register_anonymous(mymonoid, binop, name="plus_plus") - assert op.binaryop is binop - assert op.binaryop[int] is binop[int] - assert op.monoid is mymonoid - assert op.monoid[int] is mymonoid[int] + if supports_udfs: + binop = BinaryOp.register_anonymous(plus, name="plus") + mymonoid = Monoid.register_anonymous(binop, 0, name="plus") + op = Semiring.register_anonymous(mymonoid, binop, name="plus_plus") + assert op.binaryop is binop + assert op.binaryop[int] is binop[int] + assert op.monoid is mymonoid + assert op.monoid[int] is mymonoid[int] + assert op[int].parent is op assert semiring.min_plus[int].parent is semiring.min_plus - assert semiring.numpy.add_subtract[int].parent is semiring.numpy.add_subtract - assert op[int].parent is op for attr, val in vars(semiring).items(): if not isinstance(val, Semiring): @@ -881,9 +959,10 @@ def test_div_semirings(): assert result[0, 0].new() == -2 assert result.dtype == dtypes.FP64 - result = A1.T.mxm(A2, semiring.plus_floordiv).new() - assert result[0, 0].new() == -3 - assert result.dtype == dtypes.INT64 + if shouldhave(semiring, "plus_floordiv"): + result = A1.T.mxm(A2, semiring.plus_floordiv).new() + assert result[0, 0].new() == -3 + assert result.dtype == dtypes.INT64 @pytest.mark.slow @@ -902,25 +981,27 @@ def test_get_semiring(): def myplus(x, y): return x + y # pragma: no cover (numba) - binop = BinaryOp.register_anonymous(myplus, name="myplus") - st = get_semiring(monoid.plus, binop) - assert st.monoid is monoid.plus - assert st.binaryop is binop + if supports_udfs: + binop = BinaryOp.register_anonymous(myplus, name="myplus") + st = get_semiring(monoid.plus, binop) + assert st.monoid is monoid.plus + assert st.binaryop is binop - binop = BinaryOp.register_new("myplus", myplus) - assert binop is binary.myplus - st = get_semiring(monoid.plus, binop) - assert st.monoid is monoid.plus - assert st.binaryop is binop + binop = BinaryOp.register_new("myplus", myplus) + assert binop is binary.myplus + st = get_semiring(monoid.plus, binop) + assert st.monoid is monoid.plus + assert st.binaryop is binop with pytest.raises(TypeError, match="Monoid"): get_semiring(None, binary.times) with pytest.raises(TypeError, match="Binary"): get_semiring(monoid.plus, None) - sr = get_semiring(monoid.plus, binary.numpy.copysign) - assert sr.monoid is monoid.plus - assert sr.binaryop is binary.numpy.copysign + if shouldhave(binary.numpy, "copysign"): + sr = get_semiring(monoid.plus, binary.numpy.copysign) + assert sr.monoid is monoid.plus + assert sr.binaryop is binary.numpy.copysign def test_create_semiring(): @@ -958,17 +1039,22 @@ def test_commutes(): assert semiring.plus_times.is_commutative if suitesparse: assert semiring.ss.min_secondi.commutes_to is semiring.ss.min_firstj - assert semiring.plus_pow.commutes_to is semiring.plus_rpow + if shouldhave(semiring, "plus_pow") and shouldhave(semiring, "plus_rpow"): + assert semiring.plus_pow.commutes_to is semiring.plus_rpow assert not semiring.plus_pow.is_commutative - assert binary.isclose.commutes_to is binary.isclose - assert binary.isclose.is_commutative - assert binary.isclose(0.1).commutes_to is binary.isclose(0.1) - assert binary.floordiv.commutes_to is binary.rfloordiv - assert not binary.floordiv.is_commutative - assert binary.numpy.add.commutes_to is binary.numpy.add - assert binary.numpy.add.is_commutative - assert binary.numpy.less.commutes_to is binary.numpy.greater - assert not binary.numpy.less.is_commutative + if shouldhave(binary, "isclose"): + assert binary.isclose.commutes_to is binary.isclose + assert binary.isclose.is_commutative + assert binary.isclose(0.1).commutes_to is binary.isclose(0.1) + if shouldhave(binary, "floordiv") and shouldhave(binary, "rfloordiv"): + assert binary.floordiv.commutes_to is binary.rfloordiv + assert not binary.floordiv.is_commutative + if shouldhave(binary.numpy, "add"): + assert binary.numpy.add.commutes_to is binary.numpy.add + assert binary.numpy.add.is_commutative + if shouldhave(binary.numpy, "less") and shouldhave(binary.numpy, "greater"): + assert binary.numpy.less.commutes_to is binary.numpy.greater + assert not binary.numpy.less.is_commutative # Typed assert binary.plus[int].commutes_to is binary.plus[int] @@ -985,15 +1071,20 @@ def test_commutes(): assert semiring.plus_times[int].is_commutative if suitesparse: assert semiring.ss.min_secondi[int].commutes_to is semiring.ss.min_firstj[int] - assert semiring.plus_pow[int].commutes_to is semiring.plus_rpow[int] + if shouldhave(semiring, "plus_rpow"): + assert semiring.plus_pow[int].commutes_to is semiring.plus_rpow[int] assert not semiring.plus_pow[int].is_commutative - assert binary.isclose(0.1)[int].commutes_to is binary.isclose(0.1)[int] - assert binary.floordiv[int].commutes_to is binary.rfloordiv[int] - assert not binary.floordiv[int].is_commutative - assert binary.numpy.add[int].commutes_to is binary.numpy.add[int] - assert binary.numpy.add[int].is_commutative - assert binary.numpy.less[int].commutes_to is binary.numpy.greater[int] - assert not binary.numpy.less[int].is_commutative + if shouldhave(binary, "isclose"): + assert binary.isclose(0.1)[int].commutes_to is binary.isclose(0.1)[int] + if shouldhave(binary, "floordiv") and shouldhave(binary, "rfloordiv"): + assert binary.floordiv[int].commutes_to is binary.rfloordiv[int] + assert not binary.floordiv[int].is_commutative + if shouldhave(binary.numpy, "add"): + assert binary.numpy.add[int].commutes_to is binary.numpy.add[int] + assert binary.numpy.add[int].is_commutative + if shouldhave(binary.numpy, "less") and shouldhave(binary.numpy, "greater"): + assert binary.numpy.less[int].commutes_to is binary.numpy.greater[int] + assert not binary.numpy.less[int].is_commutative # Stress test (this can create extra semirings) names = dir(semiring) @@ -1014,9 +1105,12 @@ def test_from_string(): assert unary.from_string("abs[float]") is unary.abs[float] assert binary.from_string("+") is binary.plus assert binary.from_string("-[int]") is binary.minus[int] - assert binary.from_string("true_divide") is binary.numpy.true_divide - assert binary.from_string("//") is binary.floordiv - assert binary.from_string("%") is binary.numpy.mod + if config["mapnumpy"] or shouldhave(binary.numpy, "true_divide"): + assert binary.from_string("true_divide") is binary.numpy.true_divide + if shouldhave(binary, "floordiv"): + assert binary.from_string("//") is binary.floordiv + if shouldhave(binary.numpy, "mod"): + assert binary.from_string("%") is binary.numpy.mod assert monoid.from_string("*[FP64]") is monoid.times["FP64"] assert semiring.from_string("min.plus") is semiring.min_plus assert semiring.from_string("min.+") is semiring.min_plus @@ -1053,6 +1147,7 @@ def test_from_string(): agg.from_string("bad_agg") +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_lazy_op(): UnaryOp.register_new("lazy", lambda x: x, lazy=True) # pragma: no branch (numba) @@ -1115,6 +1210,7 @@ def test_positional(): assert semiring.ss.any_secondj[int].is_positional +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_udt(): record_dtype = np.dtype([("x", np.bool_), ("y", np.float64)], align=True) @@ -1280,6 +1376,7 @@ def test_binaryop_commute_exists(): raise AssertionError("Missing binaryops: " + ", ".join(sorted(missing))) +@pytest.mark.skipif("not supports_udfs") def test_binom(): v = Vector.from_coo([0, 1, 2], [3, 4, 5]) result = v.apply(binary.binom, 2).new() @@ -1341,9 +1438,11 @@ def test_is_idempotent(): assert monoid.max[int].is_idempotent assert monoid.lor.is_idempotent assert monoid.band.is_idempotent - assert monoid.numpy.gcd.is_idempotent + if shouldhave(monoid.numpy, "gcd"): + assert monoid.numpy.gcd.is_idempotent assert not monoid.plus.is_idempotent assert not monoid.times[float].is_idempotent - assert not monoid.numpy.equal.is_idempotent + if config["mapnumpy"] or shouldhave(monoid.numpy, "equal"): + assert not monoid.numpy.equal.is_idempotent with pytest.raises(AttributeError): binary.min.is_idempotent diff --git a/graphblas/tests/test_operator_types.py b/graphblas/tests/test_operator_types.py index 522b42ad2..027f02fcc 100644 --- a/graphblas/tests/test_operator_types.py +++ b/graphblas/tests/test_operator_types.py @@ -2,6 +2,7 @@ from collections import defaultdict from graphblas import backend, binary, dtypes, monoid, semiring, unary +from graphblas.core import _supports_udfs as supports_udfs from graphblas.core import operator from graphblas.dtypes import ( BOOL, @@ -83,6 +84,11 @@ BINARY[(ALL, POS)] = { "firsti", "firsti1", "firstj", "firstj1", "secondi", "secondi1", "secondj", "secondj1", } +if not supports_udfs: + udfs = {"absfirst", "abssecond", "binom", "floordiv", "rfloordiv", "rpow"} + for funcnames in BINARY.values(): + funcnames -= udfs + BINARY = {key: val for key, val in BINARY.items() if val} MONOID = { (UINT, UINT): {"band", "bor", "bxnor", "bxor"}, diff --git a/graphblas/tests/test_pickle.py b/graphblas/tests/test_pickle.py index de2d9cfda..724f43d76 100644 --- a/graphblas/tests/test_pickle.py +++ b/graphblas/tests/test_pickle.py @@ -5,6 +5,7 @@ import pytest import graphblas as gb +from graphblas.core import _supports_udfs as supports_udfs # noqa: F401 suitesparse = gb.backend == "suitesparse" @@ -36,6 +37,7 @@ def extra(): return "" +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_deserialize(extra): path = Path(__file__).parent / f"pickle1{extra}.pkl" @@ -62,6 +64,7 @@ def test_deserialize(extra): assert d3["semiring_pickle"] is gb.semiring.semiring_pickle +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_serialize(): v = gb.Vector.from_coo([1], 2) @@ -232,6 +235,7 @@ def identity_par(z): return -z +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_serialize_parameterized(): # unary_pickle = gb.core.operator.UnaryOp.register_new( @@ -285,6 +289,7 @@ def test_serialize_parameterized(): pickle.loads(pkl) # TODO: check results +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_deserialize_parameterized(extra): path = Path(__file__).parent / f"pickle2{extra}.pkl" @@ -295,6 +300,7 @@ def test_deserialize_parameterized(extra): pickle.load(f) # TODO: check results +@pytest.mark.skipif("not supports_udfs") def test_udt(extra): record_dtype = np.dtype([("x", np.bool_), ("y", np.int64)], align=True) udt = gb.dtypes.register_new("PickleUDT", record_dtype) diff --git a/graphblas/tests/test_scalar.py b/graphblas/tests/test_scalar.py index 6ee70311c..7b7c77177 100644 --- a/graphblas/tests/test_scalar.py +++ b/graphblas/tests/test_scalar.py @@ -12,7 +12,7 @@ from graphblas import backend, binary, dtypes, monoid, replace, select, unary from graphblas.exceptions import EmptyObject -from .conftest import autocompute, compute +from .conftest import autocompute, compute, pypy from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) @@ -209,12 +209,12 @@ def test_unsupported_ops(s): s[0] with pytest.raises(TypeError, match="does not support"): s[0] = 0 - with pytest.raises(TypeError, match="doesn't support"): + with pytest.raises(TypeError, match="doesn't support|does not support"): del s[0] def test_is_empty(s): - with pytest.raises(AttributeError, match="can't set attribute"): + with pytest.raises(AttributeError, match="can't set attribute|object has no setter"): s.is_empty = True @@ -226,7 +226,7 @@ def test_update(s): s << Scalar.from_value(3) assert s == 3 if s._is_cscalar: - with pytest.raises(TypeError, match="an integer is required"): + with pytest.raises(TypeError, match="an integer is required|expected integer"): s << Scalar.from_value(4.4) else: s << Scalar.from_value(4.4) @@ -358,14 +358,15 @@ def test_expr_is_like_scalar(s): } if s.is_cscalar: expected.add("_empty") - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Scalar. You may need to " "add an entry to `scalar` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " "methods, then you may need to run `python -m graphblas.core.infixmethods`." ) - assert attrs - infix_attrs == expected - assert attrs - scalar_infix_attrs == expected + assert attrs - infix_attrs - ignore == expected + assert attrs - scalar_infix_attrs - ignore == expected # Make sure signatures actually match. `expr.dup` has `**opts` skip = {"__init__", "__repr__", "_repr_html_", "dup"} for expr in [v.inner(v), v @ v, t & t]: @@ -399,7 +400,8 @@ def test_index_expr_is_like_scalar(s): } if s.is_cscalar: expected.add("_empty") - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Scalar. You may need to " "add an entry to `scalar` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " @@ -505,10 +507,10 @@ def test_scalar_expr(s): def test_sizeof(s): - if suitesparse or s._is_cscalar: + if (suitesparse or s._is_cscalar) and not pypy: assert 1 < sys.getsizeof(s) < 1000 else: - with pytest.raises(TypeError): + with pytest.raises(TypeError): # flakey coverage (why?!) sys.getsizeof(s) diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index 8505313e4..ab019b734 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -11,6 +11,7 @@ import graphblas as gb from graphblas import agg, backend, binary, dtypes, indexunary, monoid, select, semiring, unary +from graphblas.core import _supports_udfs as supports_udfs from graphblas.exceptions import ( DimensionMismatch, DomainMismatch, @@ -19,9 +20,10 @@ InvalidObject, InvalidValue, OutputNotEmpty, + UdfParseError, ) -from .conftest import autocompute, compute +from .conftest import autocompute, compute, pypy from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) @@ -798,12 +800,13 @@ def test_select_bools_and_masks(v): assert w8.isequal(w9) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_indexunary_udf(v): def twox_minusthunk(x, row, col, thunk): # pragma: no cover (numba) return 2 * x - thunk - indexunary.register_new("twox_minusthunk", twox_minusthunk) + indexunary.register_new("twox_minusthunk", twox_minusthunk, lazy=True) assert hasattr(indexunary, "twox_minusthunk") assert not hasattr(select, "twox_minusthunk") with pytest.raises(ValueError, match="SelectOp must have BOOL return type"): @@ -813,24 +816,49 @@ def twox_minusthunk(x, row, col, thunk): # pragma: no cover (numba) expected = Vector.from_coo([1, 3, 4, 6], [-2, -2, 0, -4], size=7) result = indexunary.twox_minusthunk(v, 4).new() assert result.isequal(expected) + assert pickle.loads(pickle.dumps(indexunary.triu)) is indexunary.triu + assert indexunary.twox_minusthunk[int]._numba_func(1, 2, 3, 4) == twox_minusthunk(1, 2, 3, 4) + assert indexunary.twox_minusthunk[int].orig_func is twox_minusthunk delattr(indexunary, "twox_minusthunk") def ii(x, idx, _, thunk): # pragma: no cover (numba) return idx // 2 >= thunk - select.register_new("ii", ii) - assert hasattr(indexunary, "ii") + def iin(n): + def inner(x, idx, _, thunk): # pragma: no cover (numba) + return idx // n >= thunk + + return inner + + select.register_new("ii", ii, lazy=True) + select.register_new("iin", iin, parameterized=True) + assert "ii" in dir(select) + assert "ii" in dir(indexunary) assert hasattr(select, "ii") + assert hasattr(indexunary, "ii") ii_apply = indexunary.register_anonymous(ii) expected = Vector.from_coo([1, 3, 4, 6], [False, False, True, True], size=7) result = ii_apply(v, 2).new() assert result.isequal(expected) + result = v.apply(indexunary.iin(2), 2).new() + assert result.isequal(expected) + result = v.apply(indexunary.register_anonymous(iin, parameterized=True)(2), 2).new() + assert result.isequal(expected) + ii_select = select.register_anonymous(ii) expected = Vector.from_coo([4, 6], [2, 0], size=7) result = ii_select(v, 2).new() assert result.isequal(expected) + result = v.select(select.iin(2), 2).new() + assert result.isequal(expected) + result = v.select(select.register_anonymous(iin, parameterized=True)(2), 2).new() + assert result.isequal(expected) delattr(indexunary, "ii") delattr(select, "ii") + delattr(indexunary, "iin") + delattr(select, "iin") + with pytest.raises(UdfParseError, match="Unable to parse function using Numba"): + indexunary.register_new("bad", lambda x, row, col, thunk: result) def test_reduce(v): @@ -1624,13 +1652,14 @@ def test_expr_is_like_vector(v): "resize", "update", } - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Vector. You may need to " "add an entry to `vector` or `matrix_vector` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " "methods, then you may need to run `python -m graphblas.core.infixmethods`." ) - assert attrs - infix_attrs == expected + assert attrs - infix_attrs - ignore == expected # Make sure signatures actually match skip = {"__init__", "__repr__", "_repr_html_"} for expr in [binary.times(w & w), w & w]: @@ -1672,7 +1701,8 @@ def test_index_expr_is_like_vector(v): "from_values", "resize", } - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Vector. You may need to " "add an entry to `vector` or `matrix_vector` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " @@ -1963,7 +1993,7 @@ def test_ndim(A, v): def test_sizeof(v): - if suitesparse: + if suitesparse and not pypy: assert sys.getsizeof(v) > v.nvals * 16 else: with pytest.raises(TypeError): @@ -2006,6 +2036,7 @@ def test_delete_via_scalar(v): assert v.nvals == 0 +@pytest.mark.skipif("not supports_udfs") def test_udt(): record_dtype = np.dtype([("x", np.bool_), ("y", np.float64)], align=True) udt = dtypes.register_anonymous(record_dtype, "VectorUDT") @@ -2380,6 +2411,7 @@ def test_to_coo_subset(v): assert vals.dtype == np.int64 +@pytest.mark.skipif("not supports_udfs") def test_lambda_udfs(v): result = v.apply(lambda x: x + 1).new() # pragma: no branch (numba) expected = binary.plus(v, 1).new() @@ -2506,7 +2538,8 @@ def test_from_scalar(): v = Vector.from_scalar(1, dtype="INT64[2]", size=3) w = Vector("INT64[2]", size=3) w << [1, 1] - assert v.isequal(w, check_dtype=True) + if supports_udfs: + assert v.isequal(w, check_dtype=True) def test_to_dense_from_dense(): @@ -2559,9 +2592,10 @@ def test_ss_sort(v): v.ss.sort(binary.plus) # Like compactify - _, p = v.ss.sort(lambda x, y: False, values=False) # pragma: no branch (numba) - expected_p = Vector.from_coo([0, 1, 2, 3], [1, 3, 4, 6], size=7) - assert p.isequal(expected_p) + if supports_udfs: + _, p = v.ss.sort(lambda x, y: False, values=False) # pragma: no branch (numba) + expected_p = Vector.from_coo([0, 1, 2, 3], [1, 3, 4, 6], size=7) + assert p.isequal(expected_p) # reversed _, p = v.ss.sort(binary.pair[bool], values=False) expected_p = Vector.from_coo([0, 1, 2, 3], [6, 4, 3, 1], size=7) @@ -2569,6 +2603,7 @@ def test_ss_sort(v): w, p = v.ss.sort(monoid.lxor) # Weird, but user-defined monoids may not commute, so okay +@pytest.mark.skipif("not supports_udfs") def test_subarray_dtypes(): a = np.arange(3 * 4, dtype=np.int64).reshape(3, 4) v = Vector.from_coo([1, 3, 5], a) diff --git a/graphblas/unary/numpy.py b/graphblas/unary/numpy.py index 836da2024..9b742d8bc 100644 --- a/graphblas/unary/numpy.py +++ b/graphblas/unary/numpy.py @@ -10,6 +10,7 @@ from .. import _STANDARD_OPERATOR_NAMES from .. import config as _config from .. import unary as _unary +from ..core import _supports_udfs from ..dtypes import _supports_complex _delayed = {} @@ -119,7 +120,12 @@ def __dir__(): - return globals().keys() | _delayed.keys() | _unary_names + if not _supports_udfs and not _config.get("mapnumpy"): + return globals().keys() # FLAKY COVERAGE + attrs = _delayed.keys() | _unary_names + if not _supports_udfs: + attrs &= _numpy_to_graphblas.keys() + return attrs | globals().keys() def __getattr__(name): @@ -132,6 +138,11 @@ def __getattr__(name): raise AttributeError(f"module {__name__!r} has no attribute {name!r}") if _config.get("mapnumpy") and name in _numpy_to_graphblas: globals()[name] = getattr(_unary, _numpy_to_graphblas[name]) + elif not _supports_udfs: + raise AttributeError( + f"module {__name__!r} unable to compile UDF for {name!r}; " + "install numba for UDF support" + ) else: numpy_func = getattr(_np, name) diff --git a/pyproject.toml b/pyproject.toml index 47cf1e67f..1eaa942e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3 :: Only", "Intended Audience :: Developers", "Intended Audience :: Other Audience", @@ -57,11 +58,13 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "suitesparse-graphblas >=7.4.0.0, <7.5", "numpy >=1.21", - "numba >=0.55", "donfig >=0.6", "pyyaml >=5.4", + # These won't be installed by default after 2024.3.0 + # Use e.g. "python-graphblas[suitesparse]" or "python-graphblas[default]" instead + "suitesparse-graphblas >=7.4.0.0, <7.5", + "numba >=0.55; python_version<'3.11'", # make optional where numba is not supported ] [project.urls] @@ -71,37 +74,56 @@ repository = "https://github.com/python-graphblas/python-graphblas" changelog = "https://github.com/python-graphblas/python-graphblas/releases" [project.optional-dependencies] -repr = [ - "pandas >=1.2", +suitesparse = [ + "suitesparse-graphblas >=7.4.0.0, <7.5", ] -io = [ +networkx = [ "networkx >=2.8", +] +numba = [ + "numba >=0.55", +] +pandas = [ + "pandas >=1.2", +] +scipy = [ "scipy >=1.8", +] +suitesparse-udf = [ # udf requires numba + "python-graphblas[suitesparse,numba]", +] +repr = [ + "python-graphblas[pandas]", +] +io = [ + "python-graphblas[networkx,scipy]", + "python-graphblas[numba]; python_version<'3.11'", "awkward >=1.9", - "sparse >=0.12", + "sparse >=0.13; python_version<'3.11'", # make optional, b/c sparse needs numba "fast-matrix-market >=1.4.5", ] viz = [ + "python-graphblas[networkx,scipy]", "matplotlib >=3.5", ] +datashade = [ # datashade requires numba + "python-graphblas[numba,pandas,scipy]", + "datashader >=0.12", + "hvplot >=0.7", +] test = [ - "pytest", - "packaging", - "pandas >=1.2", - "scipy >=1.8", - "tomli", + "python-graphblas[suitesparse,pandas,scipy]", + "packaging >=21", + "pytest >=6.2", + "tomli >=1", +] +default = [ + "python-graphblas[suitesparse,pandas,scipy]", + "python-graphblas[numba]; python_version<'3.11'", # make optional where numba is not supported ] complete = [ - "pandas >=1.2", - "networkx >=2.8", - "scipy >=1.8", - "awkward >=1.9", - "sparse >=0.12", - "fast-matrix-market >=1.4.5", - "matplotlib >=3.5", - "pytest", - "packaging", - "tomli", + "python-graphblas[default,io,viz,test]", + "python-graphblas[datashade]; python_version<'3.11'", # make optional, b/c datashade needs numba ] [tool.setuptools] @@ -154,8 +176,6 @@ filterwarnings = [ # See: https://docs.python.org/3/library/warnings.html#describing-warning-filters # and: https://docs.pytest.org/en/7.2.x/how-to/capture-warnings.html#controlling-warnings "error", - # MAINT: we can drop support for sparse <0.13 at any time - "ignore:`np.bool` is a deprecated alias:DeprecationWarning:sparse._umath", # sparse <0.13 # sparse 0.14.0 (2022-02-24) began raising this warning; it has been reported and fixed upstream. "ignore:coords should be an ndarray. This will raise a ValueError:DeprecationWarning:sparse._coo.core", @@ -166,6 +186,13 @@ filterwarnings = [ # And this deprecation warning was added in setuptools v67.5.0 (8 Mar 2023). See: # https://setuptools.pypa.io/en/latest/history.html#v67-5-0 "ignore:pkg_resources is deprecated as an API:DeprecationWarning:pkg_resources", + + # sre_parse deprecated in 3.11; this is triggered by awkward 0.10 + "ignore:module 'sre_parse' is deprecated:DeprecationWarning:", + "ignore:module 'sre_constants' is deprecated:DeprecationWarning:", + + # pypy gives this warning + "ignore:can't resolve package from __spec__ or __package__:ImportWarning:", ] [tool.coverage.run] diff --git a/scripts/check_versions.sh b/scripts/check_versions.sh index 54b02d1f9..026f3a656 100755 --- a/scripts/check_versions.sh +++ b/scripts/check_versions.sh @@ -4,12 +4,12 @@ # This may be helpful when updating dependency versions in CI. # Tip: add `--json` for more information. conda search 'numpy[channel=conda-forge]>=1.24.2' -conda search 'pandas[channel=conda-forge]>=1.5.3' +conda search 'pandas[channel=conda-forge]>=2.0.0' conda search 'scipy[channel=conda-forge]>=1.10.1' -conda search 'networkx[channel=conda-forge]>=3.0' -conda search 'awkward[channel=conda-forge]>=2.1.1' +conda search 'networkx[channel=conda-forge]>=3.1' +conda search 'awkward[channel=conda-forge]>=2.1.2' conda search 'sparse[channel=conda-forge]>=0.14.0' -conda search 'fast_matrix_market[channel=conda-forge]>=1.4.5' +conda search 'fast_matrix_market[channel=conda-forge]>=1.5.1' conda search 'numba[channel=conda-forge]>=0.56.4' conda search 'pyyaml[channel=conda-forge]>=6.0' conda search 'flake8-bugbear[channel=conda-forge]>=23.3.23'