Merge branch 'main' into notebooks

Author: prady0t

.github/workflows/ci.yml

@@ -11,19 +11,19 @@ jobs:
     strategy:
       matrix:
         os: ['ubuntu-latest']
-        python: ['3.10', '3.11', '3.12']
+        python: ['3.11', '3.12', '3.13']
         pip_opts: ['']
         numba_boundscheck: [0]
         include:
           - os: macos-latest
-            python: '3.10'
+            python: '3.11'
           - os: windows-latest
-            python: '3.10'
+            python: '3.11'
           - os: ubuntu-latest
-            python: '3.10'
+            python: '3.11'
             numba_boundscheck: 1
           - os: ubuntu-latest
-            python: '3.10'
+            python: '3.11'
             pip_opts: 'numpy<2'
       fail-fast: false
     runs-on: ${{ matrix.os }}

.pre-commit-config.yaml

@@ -1,14 +1,12 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
     - id: check-yaml
     - id: end-of-file-fixer
     - id: trailing-whitespace
     - id: fix-byte-order-marker
     - id: destroyed-symlinks
-    - id: fix-encoding-pragma
-      args: ["--remove"]
     - id: mixed-line-ending
     - id: name-tests-test
       args: ["--pytest-test-first"]
@@ -18,7 +16,7 @@ repos:
       exclude: ".ipynb"
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.12.4
+  rev: v0.14.1
   hooks:
     - id: ruff-check
       args: ["--fix"]

benchmarks/test_benchmark_coo.py

@@ -12,15 +12,26 @@ def format_id(format):
     return f"{format=}"
 
 
-@pytest.mark.parametrize("format", ["coo", "gcxs"])
-def test_matmul(benchmark, sides, format, rng, max_size, ids=format_id):
+@pytest.fixture(params=["coo", "gcxs"], ids=format_id)
+def format_param(request):
+    return request.param
+
+
+@pytest.fixture
+def matmul_args(sides, format_param, rng, max_size):
     m, n, p = sides
 
     if m * n >= max_size or n * p >= max_size:
         pytest.skip()
 
-    x = sparse.random((m, n), density=DENSITY, format=format, random_state=rng)
-    y = sparse.random((n, p), density=DENSITY, format=format, random_state=rng)
+    x = sparse.random((m, n), density=DENSITY, format=format_param, random_state=rng)
+    y = sparse.random((n, p), density=DENSITY, format=format_param, random_state=rng)
+
+    return x, y
+
+
+def test_matmul(benchmark, matmul_args):
+    x, y = matmul_args
 
     x @ y  # Numba compilation
 

benchmarks/test_elemwise.py

@@ -1,5 +1,4 @@
 import importlib
-import itertools
 import operator
 import os
 
@@ -28,41 +27,48 @@ def elemwise_args(request, rng, max_size):
     return s1_sps, s2_sps
 
 
-def get_elemwise_id(param):
-    f, backend = param
-    return f"{f=}-{backend=}"
+@pytest.fixture(params=[operator.add, operator.mul, operator.gt])
+def elemwise_function(request):
+    return request.param
 
 
-@pytest.fixture(
-    params=itertools.product([operator.add, operator.mul, operator.gt], ["SciPy", "Numba", "Finch"]),
-    scope="function",
-    ids=get_elemwise_id,
-)
-def backend(request):
-    f, backend = request.param
-    os.environ[sparse._ENV_VAR_NAME] = backend
+@pytest.fixture(params=["SciPy", "Numba", "Finch"])
+def backend_name(request):
+    return request.param
+
+
+@pytest.fixture
+def backend_setup(backend_name):
+    os.environ[sparse._ENV_VAR_NAME] = backend_name
     importlib.reload(sparse)
-    yield f, sparse, backend
+    yield sparse, backend_name
     del os.environ[sparse._ENV_VAR_NAME]
     importlib.reload(sparse)
 
 
-def test_elemwise(benchmark, backend, elemwise_args):
+@pytest.fixture
+def sparse_arrays(elemwise_args, backend_setup):
     s1_sps, s2_sps = elemwise_args
-    f, sparse, backend = backend
+    sparse, backend_name = backend_setup
 
-    if backend == "SciPy":
+    if backend_name == "SciPy":
         s1 = s1_sps
         s2 = s2_sps
-    elif backend == "Numba":
+    elif backend_name == "Numba":
         s1 = sparse.asarray(s1_sps)
         s2 = sparse.asarray(s2_sps)
-    elif backend == "Finch":
+    elif backend_name == "Finch":
         s1 = sparse.asarray(s1_sps.asformat("csc"), format="csc")
         s2 = sparse.asarray(s2_sps.asformat("csc"), format="csc")
 
-    f(s1, s2)
+    return s1, s2
+
+
+def test_elemwise(benchmark, elemwise_function, sparse_arrays):
+    s1, s2 = sparse_arrays
+
+    elemwise_function(s1, s2)
 
     @benchmark
     def bench():
-        f(s1, s2)
+        elemwise_function(s1, s2)

ci/environment.yml

@@ -6,7 +6,7 @@ dependencies:
   - python
   - pip
   - pip:
-    - finch-tensor>=0.2.12
+    - finch-tensor>=0.2.13
     - finch-mlir>=0.0.2
     - pytest-codspeed
     - numpy

docs/migration-jl.md

@@ -0,0 +1,26 @@
+# Migration to the Finch Julia backend
+To switch to the Finch Julia backend, set the environment variable `SPARSE_BACKEND="Finch"`, then continue using.
+
+While this is largely compatible with the Array API, support for some functions may not be present, and API compatibility isn't strictly preserved with the default (Numba) backend.
+
+However, the new backend has a large performance benefit over the default backend. Below, you will find a table of common invocations, with their equivalents in the Finch Julia backend. The most common change is a standard API for construction of arrays.
+
+| Numba Backend<br>(`SPARSE_BACKEND="Numba"`) | Finch Julia Backend<br>(`SPARSE_BACKEND="Finch"`)  | Notes |
+|---------------------------------------------|----------------------------------------------------|-------|
+| `sparse.COO.from_numpy(arr, fill_value=fv)`<br>`sparse.COO.from_scipy(arr)`<br>`sparse.COO(x)` | `sparse.asarray(x, format="coo", [fill_value=fv])` | Doesn't support pulling out individual arrays |
+| `sparse.GCXS.from_numpy(arr, fill_value=fv)`<br>`sparse.GCXS.from_scipy(arr)`<br>`sparse.GCXS(x)` | `sparse.asarray(x, format="csf", [fill_value=fv])` | Format might not be a 1:1 match |
+| `sparse.DOK.from_numpy(arr, fill_value=fv)`<br>`sparse.DOK.from_scipy(arr)`<br>`sparse.DOK(x)` | `sparse.asarray(x, format="dok", [fill_value=fv])` | Format might not be a 1:1 match |
+
+Most things work as expected, with the following exceptions, which aren't defined yet for Finch:
+
+* `sparse.broadcast_to`
+* `sparse.solve`
+* Statistical functions: `mean`, `std`, `var`
+* `sparse.isdtype`
+* `sparse.reshape`
+* Some elementwise functions
+* Manipulation functions: `concat`, `expand_dims`, `squeeze`, `flip`, `roll`, `stack`
+* `arg*` functions: `argmin`, `argmax`
+* Sorting functions: `sort`, `argsort`
+
+IEEE-754 compliance is hard to maintain with sparse arrays in general. This is now even more true of the Julia backend, which trades off performance for IEEE-754 compatibility.

mkdocs.yml

@@ -94,6 +94,7 @@ nav:
     - quickstart.md
     - construct.md
     - operations.md
+    - migration-jl.md
   - API:
     - api.md
     - api/*

pixi.toml

@@ -10,7 +10,7 @@ numba = ">=0.49"
 numpy = ">=1.17"
 
 [dependencies]
-python = ">=3.10,<3.13"
+python = ">=3.11,<3.14"
 
 [feature.extra.pypi-dependencies]
 dask = { version = ">=2024", extras = ["array"] }
@@ -55,7 +55,7 @@ juliaup = ">=1.17.10"
 
 [feature.finch.pypi-dependencies]
 scipy = ">=1.13"
-finch-tensor = ">=0.2.12"
+finch-tensor = ">=0.2.13"
 
 [feature.finch.activation.env]
 SPARSE_BACKEND = "Finch"

pyproject.toml

@@ -9,7 +9,7 @@ description = "Sparse n-dimensional arrays for the PyData ecosystem"
 readme = "README.md"
 dependencies = ["numpy>=1.17", "numba>=0.49"]
 maintainers = [{ name = "Hameer Abbasi", email = "hameerabbasi@yahoo.com" }]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 license = { file = "LICENSE" }
 keywords = ["sparse", "numpy", "scipy", "dask"]
 classifiers = [
@@ -54,7 +54,7 @@ tests = [
 tox = ["sparse[tests]", "tox"]
 notebooks = ["sparse[tests]", "nbmake", "matplotlib"]
 all = ["sparse[docs,tox,notebooks,mlir]", "matrepr"]
-finch = ["finch-tensor>=0.2.12"]
+finch = ["finch-tensor>=0.2.13"]
 mlir = ["finch-mlir>=0.0.2"]
 
 [project.urls]

sparse/numba_backend/_coo/core.py

@@ -596,12 +596,12 @@ def nnz(self):
         --------
         >>> x = np.array([0, 0, 1, 0, 1, 2, 0, 1, 2, 3, 0, 0])
         >>> np.count_nonzero(x)
-        6
+        np.int64(6)
         >>> s = COO.from_numpy(x)
         >>> s.nnz
         6
         >>> np.count_nonzero(x) == s.nnz
-        True
+        np.True_
         """
         return self.coords.shape[1]