🧱 static integration testing (#35)

* 🧑‍💻 lefthook config fine-tuning
* 🎨 remove redundant `bound=object`
* 🚚 restructured tests directories
* 🔧⬆️ mypy config tweaks, reorder dep groups, and bump deps
* 💡 remove redundant `# noqa`
* 🧪 add (failing) numpy integration test for `HasArrayNamespace`
* 🔧 fix awkward dependency groups
* 👷 integration testing matrix for numpy
* 🙈 ignore some irrelevant ruff codes for the static integration tests
* 💚 fix test path
* 💚 don't use `--frozen` when installing different a numpy version
* ⬆️ might as well bump `setup-uv` 
* 💚 clean up CI debug statements
* 🐛 fix `HasArrayNamespace` falsely rejecting `ndarray` instances on numpy 2
* 🧙 split numpy 1 and 2 integration tests with magic
* ✂️ don't `cut` more than needed
* 🩹 don't attempt to directly use `np.array_api.Array`
* 🔧 move ruff ignore rules for the tests to the tests
* ⏪ temporarily restore the lefthook step

Author: jorenham

.github/workflows/ci.yml

@@ -12,83 +12,98 @@ concurrency:
   cancel-in-progress: true
 
 env:
+  UV_LOCKED: 1
   # Many color libraries just need this to be set to any value, but at least
   # one distinguishes color depth, where "3" -> "256-bit color".
   FORCE_COLOR: 3
 
 jobs:
-  format:
-    name: Format
+  lint:
+    name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install uv
-        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba
+        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
 
-      - name: Install the project
-        run: uv sync --locked --group test
+      - name: ruff
+        run: |
+          uv run ruff check --output-format=github
+          uv run ruff format --check
 
-      - name: Run lefthook hooks
-        run: uv run --frozen lefthook run pre-commit
+      # TODO: Fail if lefthook changes any files:
+      # https://github.com/data-apis/array-api-typing/pull/35/files#r2179941334
+      - name: lefthook
+        run: uv run lefthook run pre-commit
 
-  checks:
-    name: Check Python ${{ matrix.python-version }} on ${{ matrix.runs-on }}
+      - name: mypy
+        run: uv run mypy --tb --no-incremental --cache-dir=/dev/null src
+
+      # TODO: (based)pyright
+
+  test_runtime:
+    name: runtime tests
     runs-on: ${{ matrix.runs-on }}
-    needs: [format]
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.11", "3.12", "3.13"]
         runs-on: [ubuntu-latest, macos-latest, windows-latest]
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install uv
-        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba
+        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install the project
-        run: uv sync --locked --group test
-
       - name: Test package
         run: >-
-          uv run --frozen pytest
-          --cov --cov-report=xml --cov-report=term --durations=20
+          uv run --group=test_runtime
+          pytest --cov --cov-report=xml --cov-report=term --durations=20
 
       - name: Upload coverage report
-        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24
+        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  check_oldest:
-    name: Check Oldest Dependencies
-    runs-on: ${{ matrix.runs-on }}
-    needs: [format]
+  test_integration_numpy:
+    name: integration tests (numpy)
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.11"]
-        runs-on: [ubuntu-latest]
+        numpy-version: ["1.25.0", "1.26.4", "2.0.2", "2.1.3", "2.2.6", "2.3.1"]
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install the project
-        run: uv sync --group test --resolution lowest-direct
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - name: Test package
-        run: >-
-          uv run --frozen pytest
-          --cov --cov-report=xml --cov-report=term --durations=20
-
-      - name: Upload coverage report
-        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24
+      - uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
+          python-version: "3.11"
+          activate-environment: true
+
+      - name: get major numpy version
+        id: numpy-major
+        run: |
+          version=$(echo ${{ matrix.numpy-version }} | cut -c 1)
+          echo "::set-output name=version::$version"
+
+      - name: install deps
+        run: |
+          uv sync --no-editable --group=mypy
+          uv pip install numpy==${{ matrix.numpy-version }}
+
+      # NOTE: `uv run --with=...` will be ignored by mypy (and `--isolated` does not help)
+      - name: mypy
+        run: >
+          uv run --no-sync --active
+          mypy --tb --no-incremental --cache-dir=/dev/null
+          tests/integration/test_numpy${{ steps.numpy-major.outputs.version }}.pyi
+
+      # TODO: (based)pyright
+
+  # TODO: integration tests for array-api-strict
+  # TODO: integration tests for 3rd party libs such as cupy, pytorch, tensorflow, dask, etc.

lefthook.yml

@@ -1,16 +1,33 @@
 #   Refer for explanation to following link:
 #   https://lefthook.dev/configuration/
-#
+
+templates:
+  run: run --no-sync
 
 pre-commit:
   parallel: true
   jobs:
-    - name: ruff-fix
-      glob: "*.py"
-      run: uv run ruff check --fix {staged_files}
-    - name: ruff-format
-      glob: "*.py"
-      run: uv run ruff format {staged_files}
+    - name: ruff
+      glob: "*.{py,pyi}"
+      stage_fixed: true
+      group:
+        piped: true
+        jobs:
+          - name: check
+            run: uv {run} ruff check --fix {staged_files}
+          - name: format
+            run: uv {run} ruff format {staged_files}
     - name: mypy
-      glob: "*.py"
-      run: uv run --group mypy mypy {staged_files}
+      glob: "*.{py,pyi}"
+      run: uv {run} mypy {staged_files}
+
+post-checkout:
+  jobs:
+    - run: uv sync
+      glob: uv.lock
+
+post-merge:
+  files: "git diff-tree -r --name-only --no-commit-id ORIG_HEAD HEAD"
+  jobs:
+    - run: uv sync
+      glob: uv.lock

pyproject.toml

@@ -41,18 +41,27 @@
 
 [dependency-groups]
   dev = [
-    { include-group = "test" },
-    "lefthook>=1.11.13",
+    { include-group = "lint" },
+    { include-group = "mypy" },
+    { include-group = "test_runtime" },
+    { include-group = "test_numpy" },
+    "lefthook==1.11.14",
+    "orjson>=3.10.18; python_version<'3.14'", # used by mypy
   ]
-  test = [
-    "numpy>=1.25",
-    "pytest==8.3.3",
-    "pytest-cov>=6",
-    "pytest-github-actions-annotate-failures>=0.3.0",
-    "sybil>=8.0.0",
+  lint = [
+    "ruff==0.12.1",
   ]
   mypy = [
-    "mypy>=1.16.0"
+    "mypy==1.16.1",
+  ]
+  test_runtime = [
+    "pytest==8.4.1",
+    "pytest-cov>=6.2.1",
+    "pytest-github-actions-annotate-failures==0.3.0",
+    "sybil==9.1.0",
+  ]
+  test_numpy = [
+    "numpy>=1.25",
   ]
 
 
@@ -75,28 +84,14 @@ version_tuple = {version_tuple!r}
 
 
 [tool.mypy]
-  files = ["src", "tests"]
-  python_version = "3.10"
-  mypy_path = "src"
+  mypy_path = ["src"]
+  namespace_packages = true
 
   strict = true
-  disallow_incomplete_defs = true
-  disallow_untyped_defs = true
-  disable_bytearray_promotion = true  # Note(2024-12-05): these are private flags
-  disable_memoryview_promotion = true  # Note(2024-12-05): these are private flags
+  allow_redefinition_new = true
+  local_partial_types = true
   enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
-
-  warn_return_any = true
   warn_unreachable = true
-  warn_unused_configs = true
-
-  [[tool.mypy.overrides]]
-    module = "sybil.*"
-    ignore_missing_imports = true
-
-  [[tool.mypy.overrides]]
-    module = "tests.*"
-    disallow_untyped_defs = false
 
 
 [tool.pytest.ini_options]
@@ -114,9 +109,8 @@ version_tuple = {version_tuple!r}
     "ignore:ast\\.Str is deprecated and will be removed in Python 3\\.14:DeprecationWarning",
   ]
   log_cli_level = "INFO"
-  minversion = "8.3"
-  testpaths = ["README.md", "src/", "tests/"]
-  norecursedirs = ["docs/_build"]
+  minversion = "8.4"
+  testpaths = ["README.md", "src/", "tests/runtime/"]
   xfail_strict = true
 
 
@@ -138,8 +132,15 @@ version_tuple = {version_tuple!r}
       "ISC001", # Conflicts with formatter
     ]
 
-    [tool.ruff.lint.per-file-ignores]
-      "tests/*.py" = ["ANN201", "D1", "S101"]
+    [tool.ruff.lint.pylint]
+      allow-dunder-method-names = [
+        "__array_api_version__",
+        "__array_namespace__",
+        "__array_namespace_info__",
+        "__dlpack__",
+        "__dlpack_device__",
+        "__dlpack_device__",
+      ]
 
     [tool.ruff.lint.flake8-import-conventions]
       banned-from = ["array_api_typing"]
@@ -149,7 +150,7 @@ version_tuple = {version_tuple!r}
 
     [tool.ruff.lint.isort]
       combine-as-imports = true
-      extra-standard-library = ["typing_extensions"]
+      extra-standard-library = ["_typeshed", "typing_extensions"]
       known-local-folder = ["array_api_typing"]
 
   [tool.ruff.format]

src/array_api_typing/_namespace.py

@@ -1,10 +1,10 @@
 __all__ = ("HasArrayNamespace",)
 
 from types import ModuleType
-from typing import Protocol
+from typing import Literal, Protocol
 from typing_extensions import TypeVar
 
-T_co = TypeVar("T_co", covariant=True, bound=object, default=ModuleType)
+T_co = TypeVar("T_co", covariant=True, default=ModuleType)
 
 
 class HasArrayNamespace(Protocol[T_co]):
@@ -25,4 +25,6 @@ class HasArrayNamespace(Protocol[T_co]):
 
     """
 
-    def __array_namespace__(self, /, *, api_version: str | None = None) -> T_co: ...  # noqa: PLW3201
+    def __array_namespace__(
+        self, /, *, api_version: Literal["2021.12"] | None = None
+    ) -> T_co: ...

tests/.ruff.toml

@@ -0,0 +1,9 @@
+extend = "../pyproject.toml"
+
+[lint]
+extend-ignore = [
+  "ANN201", # https://docs.astral.sh/ruff/rules/missing-return-type-undocumented-public-function/
+  "D1",     # https://docs.astral.sh/ruff/rules/#pydocstyle-d
+  "INP001", # https://docs.astral.sh/ruff/rules/implicit-namespace-package/
+  "S101",   # https://docs.astral.sh/ruff/rules/assert/
+]

tests/__init__.py

@@ -0,0 +1,4 @@
+extend = "../.ruff.toml"
+
+[lint]
+extend-ignore = ["B018", "PYI015", "PYI017"]

tests/integration/.ruff.toml

@@ -0,0 +1,12 @@
+from typing import Any
+
+# requires numpy < 2
+import numpy.array_api as np
+
+import array_api_typing as xpt
+
+###
+# Ensure that `np.ndarray` instances are assignable to `xpt.HasArrayNamespace`.
+
+arr = np.eye(2)
+arr_namespace: xpt.HasArrayNamespace[Any] = arr

tests/integration/test_numpy1.pyi

@@ -0,0 +1,11 @@
+from typing import Any
+
+import numpy.typing as npt
+
+import array_api_typing as xpt
+
+###
+# Ensure that `np.ndarray` instances are assignable to `xpt.HasArrayNamespace`.
+
+arr: npt.NDArray[Any]
+arr_namespace: xpt.HasArrayNamespace[Any] = arr

tests/integration/test_numpy2.pyi

@@ -15,31 +15,31 @@ class CheckableHasArrayNamespace(xpt.HasArrayNamespace, Protocol):
 class GoodArray:
     """Example class that implements the HasArrayNamespace protocol."""
 
-    def __array_namespace__(self) -> object:  # noqa: PLW3201
+    def __array_namespace__(self) -> object:
         return SimpleNamespace()
 
 
 class BadArray:
     """Example class that does not implement the HasArrayNamespace protocol."""
 
 
-def test_has_namespace_class():
+def test_has_namespace_class() -> None:
     """Test that GoodArray is a subclass of HasArrayNamespace."""
     assert issubclass(GoodArray, CheckableHasArrayNamespace)
 
 
-def test_has_namespace_instance():
+def test_has_namespace_instance() -> None:
     """Test that an instance of GoodArray is recognized as HasArrayNamespace."""
     x = GoodArray()
     assert isinstance(x, CheckableHasArrayNamespace)
 
 
-def test_not_has_namespace_class():
+def test_not_has_namespace_class() -> None:
     """Test that BadArray is not a subclass of HasArrayNamespace."""
     assert not issubclass(BadArray, CheckableHasArrayNamespace)
 
 
-def test_not_has_namespace_instance():
+def test_not_has_namespace_instance() -> None:
     """Test that an instance of BadArray is not recognized as HasArrayNamespace."""
     y = BadArray()
     assert not isinstance(y, CheckableHasArrayNamespace)