From 0f70b037aff5284b15601d5683b861a0df7496d1 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 22:50:48 -0400 Subject: [PATCH 01/22] ignore types folder --- src/scyjava/types/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/scyjava/types/.gitignore diff --git a/src/scyjava/types/.gitignore b/src/scyjava/types/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/src/scyjava/types/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore From 459a9992539a193b11ceccb625e938218a26b7eb Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:17:04 -0400 Subject: [PATCH 02/22] Add scyjava-stubs CLI and dynamic import functionality - Introduced `scyjava-stubs` executable for generating Python type stubs from Java classes. - Implemented dynamic import logic in `_dynamic_import.py`. - Added stub generation logic in `_genstubs.py`. - Updated `pyproject.toml` to include new dependencies and scripts. - Created `__init__.py` for the `_stubs` package to expose key functionalities. --- pyproject.toml | 4 + src/scyjava/_stubs/__init__.py | 4 + src/scyjava/_stubs/_cli.py | 152 +++++++++++++++++++++ src/scyjava/_stubs/_dynamic_import.py | 61 +++++++++ src/scyjava/_stubs/_genstubs.py | 184 ++++++++++++++++++++++++++ 5 files changed, 405 insertions(+) create mode 100644 src/scyjava/_stubs/__init__.py create mode 100644 src/scyjava/_stubs/_cli.py create mode 100644 src/scyjava/_stubs/_dynamic_import.py create mode 100644 src/scyjava/_stubs/_genstubs.py diff --git a/pyproject.toml b/pyproject.toml index 8da9830..b36f2bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "jpype1 >= 1.3.0", "jgo", "cjdk", + "stubgenj", ] [project.optional-dependencies] @@ -53,6 +54,9 @@ dev = [ "validate-pyproject[all]" ] +[project.scripts] +scyjava-stubgen = "scyjava._stubs._cli:main" + [project.urls] homepage = "https://github.com/scijava/scyjava" documentation = "https://github.com/scijava/scyjava/blob/main/README.md" diff --git a/src/scyjava/_stubs/__init__.py b/src/scyjava/_stubs/__init__.py new file mode 100644 index 0000000..4f42a0b --- /dev/null +++ b/src/scyjava/_stubs/__init__.py @@ -0,0 +1,4 @@ +from ._dynamic_import import dynamic_import +from ._genstubs import generate_stubs + +__all__ = ["dynamic_import", "generate_stubs"] diff --git a/src/scyjava/_stubs/_cli.py b/src/scyjava/_stubs/_cli.py new file mode 100644 index 0000000..eeb6970 --- /dev/null +++ b/src/scyjava/_stubs/_cli.py @@ -0,0 +1,152 @@ +"""The scyjava-stubs executable.""" + +import argparse +import importlib +import importlib.util +import logging +import sys +from pathlib import Path + +from ._genstubs import generate_stubs + + +def main() -> None: + """The main entry point for the scyjava-stubs executable.""" + logging.basicConfig(level="INFO") + parser = argparse.ArgumentParser( + description="Generate Python Type Stubs for Java classes." + ) + parser.add_argument( + "endpoints", + type=str, + nargs="+", + help="Maven endpoints to install and use (e.g. org.myproject:myproject:1.0.0)", + ) + parser.add_argument( + "--prefix", + type=str, + help="package prefixes to generate stubs for (e.g. org.myproject), " + "may be used multiple times. If not specified, prefixes are gleaned from the " + "downloaded artifacts.", + action="append", + default=[], + metavar="PREFIX", + dest="prefix", + ) + path_group = parser.add_mutually_exclusive_group() + path_group.add_argument( + "--output-dir", + type=str, + default=None, + help="Filesystem path to write stubs to.", + ) + path_group.add_argument( + "--output-python-path", + type=str, + default=None, + help="Python path to write stubs to (e.g. 'scyjava.types').", + ) + parser.add_argument( + "--convert-strings", + dest="convert_strings", + action="store_true", + default=False, + help="convert java.lang.String to python str in return types. " + "consult the JPype documentation on the convertStrings flag for details", + ) + parser.add_argument( + "--no-javadoc", + dest="with_javadoc", + action="store_false", + default=True, + help="do not generate docstrings from JavaDoc where available", + ) + + rt_group = parser.add_mutually_exclusive_group() + rt_group.add_argument( + "--runtime-imports", + dest="runtime_imports", + action="store_true", + default=True, + help="Add runtime imports to the generated stubs. ", + ) + rt_group.add_argument( + "--no-runtime-imports", dest="runtime_imports", action="store_false" + ) + + parser.add_argument( + "--remove-namespace-only-stubs", + dest="remove_namespace_only_stubs", + action="store_true", + default=False, + help="Remove stubs that export no names beyond a single __module_protocol__. " + "This leaves some folders as PEP420 implicit namespace folders.", + ) + + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + + args = parser.parse_args() + output_dir = _get_ouput_dir(args.output_dir, args.output_python_path) + if not output_dir.exists(): + output_dir.mkdir(parents=True, exist_ok=True) + + generate_stubs( + endpoints=args.endpoints, + prefixes=args.prefix, + output_dir=output_dir, + convert_strings=args.convert_strings, + include_javadoc=args.with_javadoc, + add_runtime_imports=args.runtime_imports, + remove_namespace_only_stubs=args.remove_namespace_only_stubs, + ) + + +def _get_ouput_dir(output_dir: str | None, python_path: str | None) -> Path: + if out_dir := output_dir: + return Path(out_dir) + if pp := python_path: + return _glean_path(pp) + try: + import scyjava + + return Path(scyjava.__file__).parent / "types" + except ImportError: + return Path("stubs") + + +def _glean_path(pp: str) -> Path: + try: + importlib.import_module(pp.split(".")[0]) + except ModuleNotFoundError: + # the top level module doesn't exist: + raise ValueError(f"Module {pp} does not exist. Cannot install stubs there.") + + try: + spec = importlib.util.find_spec(pp) + except ModuleNotFoundError as e: + # at least one of the middle levels doesn't exist: + raise NotImplementedError(f"Cannot install stubs to {pp}: {e}") + + new_ns = None + if not spec: + # if we get here, it means everything but the last level exists: + parent, new_ns = pp.rsplit(".", 1) + spec = importlib.util.find_spec(parent) + + if not spec: + # if we get here, it means the last level doesn't exist: + raise ValueError(f"Module {pp} does not exist. Cannot install stubs there.") + + search_locations = spec.submodule_search_locations + if not spec.loader and search_locations: + # namespace package with submodules + return Path(search_locations[0]) + if spec.origin: + return Path(spec.origin).parent + if new_ns and search_locations: + # namespace package with submodules + return Path(search_locations[0]) / new_ns + + raise ValueError(f"Error finding module {pp}. Cannot install stubs there.") diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py new file mode 100644 index 0000000..af50065 --- /dev/null +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -0,0 +1,61 @@ +import ast +from logging import warning +from pathlib import Path +from typing import Any, Callable + + +def dynamic_import( + module_name: str, module_file: str, *endpoints: str +) -> tuple[list[str], Callable[[str], Any]]: + import scyjava + import scyjava.config + + for ep in endpoints: + if ep not in scyjava.config.endpoints: + scyjava.config.endpoints.append(ep) + + module_all = [] + try: + my_stub = Path(module_file).with_suffix(".pyi") + stub_ast = ast.parse(my_stub.read_text()) + module_all = sorted( + { + node.name + for node in stub_ast.body + if isinstance(node, ast.ClassDef) and not node.name.startswith("__") + } + ) + except (OSError, SyntaxError): + warning( + f"Failed to read stub file {my_stub!r}. Falling back to empty __all__.", + stacklevel=3, + ) + + def module_getattr(name: str, mod_name: str = module_name) -> Any: + if module_all and name not in module_all: + raise AttributeError(f"module {module_name!r} has no attribute {name!r}") + + # this strip is important... and tricky, because it depends on the + # namespace that we intend to install the stubs into. + install_path = "scyjava.types." + if mod_name.startswith(install_path): + mod_name = mod_name[len(install_path) :] + + full_name = f"{mod_name}.{name}" + + class ProxyMeta(type): + def __repr__(self) -> str: + return f"" + + class Proxy(metaclass=ProxyMeta): + def __new__(_cls_, *args: Any, **kwargs: Any) -> Any: + cls = scyjava.jimport(full_name) + return cls(*args, **kwargs) + + Proxy.__name__ = name + Proxy.__qualname__ = name + Proxy.__module__ = module_name + Proxy.__doc__ = f"Proxy for {full_name}" + return Proxy + + return module_all, module_getattr diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py new file mode 100644 index 0000000..be800bd --- /dev/null +++ b/src/scyjava/_stubs/_genstubs.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import ast +import logging +import os +import shutil +import subprocess +from importlib import import_module +from itertools import chain +from pathlib import Path, PurePath +from typing import TYPE_CHECKING, Any +from unittest.mock import patch +from zipfile import ZipFile + +import cjdk +import scyjava +import scyjava.config +import stubgenj + +if TYPE_CHECKING: + from collections.abc import Sequence + +logger = logging.getLogger(__name__) + + +def generate_stubs( + endpoints: Sequence[str], + prefixes: Sequence[str] = (), + output_dir: str | Path = "stubs", + convert_strings: bool = True, + include_javadoc: bool = True, + add_runtime_imports: bool = True, + remove_namespace_only_stubs: bool = False, +) -> None: + """Generate stubs for the given maven endpoints. + + Parameters + ---------- + endpoints : Sequence[str] + The maven endpoints to generate stubs for. This should be a list of GAV + coordinates, e.g. ["org.apache.commons:commons-lang3:3.12.0"]. + prefixes : Sequence[str], optional + The prefixes to generate stubs for. This should be a list of Java class + prefixes that you expect to find in the endpoints. For example, + ["org.apache.commons"]. If not provided, the prefixes will be + automatically determined from the jar files provided by endpoints. + output_dir : str | Path, optional + The directory to write the generated stubs to. Defaults to "stubs". + convert_strings : bool, optional + Whether to cast Java strings to Python strings in the stubs. Defaults to True. + NOTE: This leads to type stubs that may not be strictly accurate at runtime. + The actual runtime type of strings is determined by whether jpype.startJVM is + called with the `convertStrings` argument set to True or False. By setting + this `convert_strings` argument to true, the type stubs will be generated as if + `convertStrings` is set to True: that is, all string types will be listed as + `str` rather than `java.lang.String | str`. This is a safer default (as `str`) + is a subtype of `java.lang.String`), but may lead to type errors in some cases. + include_javadoc : bool, optional + Whether to include Javadoc in the generated stubs. Defaults to True. + add_runtime_imports : bool, optional + Whether to add runtime imports to the generated stubs. Defaults to True. + This is useful if you want to use the stubs as a runtime package with type + safety. + remove_namespace_only_stubs : bool, optional + Whether to remove stubs that export no names beyond a single + `__module_protocol__`. This leaves some folders as PEP420 implicit namespace + folders. Defaults to False. Setting this to `True` is useful if you want to + merge the generated stubs with other stubs in the same namespace. Without this, + the `__init__.pyi` for any given module will be whatever whatever the *last* + stub generator wrote to it (and therefore inaccurate). + """ + import jpype + import jpype.imports + + startJVM = jpype.startJVM + + scyjava.config.endpoints.extend(endpoints) + + def _patched_start(*args: Any, **kwargs: Any) -> None: + kwargs.setdefault("convertStrings", convert_strings) + startJVM(*args, **kwargs) + + with patch.object(jpype, "startJVM", new=_patched_start): + scyjava.start_jvm() + + _prefixes = set(prefixes) + if not _prefixes: + cp = jpype.getClassPath(env=False) + ep_artifacts = tuple(ep.split(":")[1] for ep in endpoints) + for j in cp.split(os.pathsep): + if Path(j).name.startswith(ep_artifacts): + _prefixes.update(list_top_level_packages(j)) + + prefixes = sorted(_prefixes) + logger.info(f"Using endpoints: {scyjava.config.endpoints!r}") + logger.info(f"Generating stubs for: {prefixes}") + logger.info(f"Writing stubs to: {output_dir}") + + jmodules = [import_module(prefix) for prefix in prefixes] + stubgenj.generateJavaStubs( + jmodules, + useStubsSuffix=False, + outputDir=str(output_dir), + jpypeJPackageStubs=False, + includeJavadoc=include_javadoc, + ) + + output_dir = Path(output_dir) + if add_runtime_imports: + logger.info("Adding runtime imports to generated stubs") + for stub in output_dir.rglob("*.pyi"): + stub_ast = ast.parse(stub.read_text()) + members = {node.name for node in stub_ast.body if hasattr(node, "name")} + if members == {"__module_protocol__"}: + # this is simply a module stub... no exports + if remove_namespace_only_stubs: + logger.info("Removing namespace only stub %s", stub) + stub.unlink() + continue + if add_runtime_imports: + real_import = stub.with_suffix(".py") + endpoint_args = ", ".join(repr(x) for x in endpoints) + real_import.write_text(INIT_TEMPLATE.format(endpoints=endpoint_args)) + + ruff_check(output_dir.absolute()) + + +# the "real" init file that goes into the stub package +INIT_TEMPLATE = """\ +# this file was autogenerated by scyjava-stubgen +# it creates a __getattr__ function that will dynamically import +# the requested class from the Java namespace corresponding to this module. +# see scyjava._stubs for implementation details. +from scyjava._stubs import dynamic_import + +__all__, __getattr__ = dynamic_import(__name__, __file__, {endpoints}) +""" + + +def ruff_check(output: Path, select: str = "E,W,F,I,UP,C4,B,RUF,TC,TID") -> None: + """Run ruff check and format on the generated stubs.""" + if not shutil.which("ruff"): + return + + py_files = [str(x) for x in chain(output.rglob("*.py"), output.rglob("*.pyi"))] + logger.info( + "Running ruff check on %d generated stubs in % s", + len(py_files), + str(output), + ) + subprocess.run( + [ + "ruff", + "check", + *py_files, + "--quiet", + "--fix-only", + "--unsafe-fixes", + f"--select={select}", + ] + ) + logger.info("Running ruff format") + subprocess.run(["ruff", "format", *py_files, "--quiet"]) + + +def list_top_level_packages(jar_path: str) -> set[str]: + """Inspect a JAR file and return the set of top-level Java package names.""" + packages: set[str] = set() + with ZipFile(jar_path, "r") as jar: + # find all classes + class_dirs = { + entry.parent + for x in jar.namelist() + if (entry := PurePath(x)).suffix == ".class" + } + + roots: set[PurePath] = set() + for p in sorted(class_dirs, key=lambda p: len(p.parts)): + # If none of the already accepted roots is a parent of p, keep p + if not any(root in p.parents for root in roots): + roots.add(p) + packages.update({str(p).replace(os.sep, ".") for p in roots}) + + return packages From a8b2da72273ddeffde466107ea8ba8daeba259ab Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:17:25 -0400 Subject: [PATCH 03/22] remove import --- src/scyjava/_stubs/_genstubs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index be800bd..74ac78c 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -12,7 +12,6 @@ from unittest.mock import patch from zipfile import ZipFile -import cjdk import scyjava import scyjava.config import stubgenj From 686739b56c0645e1dc57f129169098a283f63e58 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:46:32 -0400 Subject: [PATCH 04/22] add test --- src/scyjava/_stubs/_genstubs.py | 4 +++ tests/test_stubgen.py | 56 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/test_stubgen.py diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 74ac78c..b2c1910 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -69,6 +69,10 @@ def generate_stubs( stub generator wrote to it (and therefore inaccurate). """ import jpype + + # FIXME: either remove the _JImportLoader from sys.meta_path after this is done + # (if it wasn't there to begin with), or replace the import_module calls below + # with a more direct JPackage call import jpype.imports startJVM = jpype.startJVM diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py new file mode 100644 index 0000000..4b7a0bb --- /dev/null +++ b/tests/test_stubgen.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING +from unittest.mock import patch + +import jpype +import jpype.imports + +import scyjava +from scyjava._stubs import _cli + +if TYPE_CHECKING: + from pathlib import Path + + import pytest + + +def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "scyjava-stubgen", + "org.scijava:parsington:3.1.0", + "--output-dir", + str(tmp_path), + ], + ) + _cli.main() + + # remove the `jpype.imports` magic from the import system if present + mp = [x for x in sys.meta_path if not isinstance(x, jpype.imports._JImportLoader)] + monkeypatch.setattr(sys, "meta_path", mp) + + # add tmp_path to the import path + monkeypatch.setattr(sys, "path", [str(tmp_path)]) + + # first cleanup to make sure we are not importing from the cache + sys.modules.pop("org", None) + sys.modules.pop("org.scijava", None) + sys.modules.pop("org.scijava.parsington", None) + # make sure the stubgen command works and that we can now impmort stuff + + with patch.object(scyjava._jvm, "start_jvm") as mock_start_jvm: + from org.scijava.parsington import Function + + assert Function is not None + assert repr(Function) == "" + # ensure that no calls to start_jvm were made + mock_start_jvm.assert_not_called() + + # only after instantiating the class should we have a call to start_jvm + func = Function(1) + mock_start_jvm.assert_called_once() + assert isinstance(func, jpype.JObject) From 7fe31af6f6d0c72bd9ae1bfb8a77ee44a85033a8 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:47:27 -0400 Subject: [PATCH 05/22] add comment to clarify stubgen command execution in test --- tests/test_stubgen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py index 4b7a0bb..bb7fd9b 100644 --- a/tests/test_stubgen.py +++ b/tests/test_stubgen.py @@ -17,6 +17,7 @@ def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # run the stubgen command as if it was run from the command line monkeypatch.setattr( sys, "argv", From afcc7a7bb30da19cc965e3079f4a32be517471df Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:57:34 -0400 Subject: [PATCH 06/22] refactor: clean up jpype imports in stubgen test and main module --- src/scyjava/_stubs/_genstubs.py | 19 +++++++++++++------ tests/test_stubgen.py | 1 - 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index b2c1910..b7d5b46 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -8,10 +8,13 @@ from importlib import import_module from itertools import chain from pathlib import Path, PurePath +import sys from typing import TYPE_CHECKING, Any from unittest.mock import patch from zipfile import ZipFile +import jpype + import scyjava import scyjava.config import stubgenj @@ -70,11 +73,6 @@ def generate_stubs( """ import jpype - # FIXME: either remove the _JImportLoader from sys.meta_path after this is done - # (if it wasn't there to begin with), or replace the import_module calls below - # with a more direct JPackage call - import jpype.imports - startJVM = jpype.startJVM scyjava.config.endpoints.extend(endpoints) @@ -99,7 +97,16 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: logger.info(f"Generating stubs for: {prefixes}") logger.info(f"Writing stubs to: {output_dir}") - jmodules = [import_module(prefix) for prefix in prefixes] + metapath = sys.meta_path + try: + import jpype.imports + + jmodules = [import_module(prefix) for prefix in prefixes] + finally: + # remove the jpype.imports magic from the import system + # if it wasn't there to begin with + sys.meta_path = metapath + stubgenj.generateJavaStubs( jmodules, useStubsSuffix=False, diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py index bb7fd9b..300ee7f 100644 --- a/tests/test_stubgen.py +++ b/tests/test_stubgen.py @@ -5,7 +5,6 @@ from unittest.mock import patch import jpype -import jpype.imports import scyjava from scyjava._stubs import _cli From a5cacc809e0db42f31f44646d4646f176894fe79 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:58:39 -0400 Subject: [PATCH 07/22] remove unused jpype import from _genstubs.py --- src/scyjava/_stubs/_genstubs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index b7d5b46..22c0eca 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -13,8 +13,6 @@ from unittest.mock import patch from zipfile import ZipFile -import jpype - import scyjava import scyjava.config import stubgenj From ab1bc2d6579999deb572ee13288d86a3e24321af Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:59:13 -0400 Subject: [PATCH 08/22] fix: add future annotations import to _cli.py --- src/scyjava/_stubs/_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scyjava/_stubs/_cli.py b/src/scyjava/_stubs/_cli.py index eeb6970..3a4d7df 100644 --- a/src/scyjava/_stubs/_cli.py +++ b/src/scyjava/_stubs/_cli.py @@ -1,5 +1,7 @@ """The scyjava-stubs executable.""" +from __future__ import annotations + import argparse import importlib import importlib.util From 11649faea090a8a65dc92f2eaa853e3c5f6249b7 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 23 Apr 2025 08:16:30 -0400 Subject: [PATCH 09/22] refactor: enhance dynamic_import function to accept base_prefix and improve stub generation --- src/scyjava/_stubs/_dynamic_import.py | 23 ++++++++++++----------- src/scyjava/_stubs/_genstubs.py | 17 ++++++++++++++--- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py index af50065..f62407b 100644 --- a/src/scyjava/_stubs/_dynamic_import.py +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -1,11 +1,14 @@ import ast from logging import warning from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Sequence def dynamic_import( - module_name: str, module_file: str, *endpoints: str + module_name: str, + module_file: str, + endpoints: Sequence[str] = (), + base_prefix: str = "", ) -> tuple[list[str], Callable[[str], Any]]: import scyjava import scyjava.config @@ -35,27 +38,25 @@ def module_getattr(name: str, mod_name: str = module_name) -> Any: if module_all and name not in module_all: raise AttributeError(f"module {module_name!r} has no attribute {name!r}") - # this strip is important... and tricky, because it depends on the - # namespace that we intend to install the stubs into. - install_path = "scyjava.types." - if mod_name.startswith(install_path): - mod_name = mod_name[len(install_path) :] + # cut the mod_name to only the part including the base_prefix and after + if base_prefix in mod_name: + mod_name = mod_name[mod_name.index(base_prefix) :] - full_name = f"{mod_name}.{name}" + class_path = f"{mod_name}.{name}" class ProxyMeta(type): def __repr__(self) -> str: - return f"" + return f"" class Proxy(metaclass=ProxyMeta): def __new__(_cls_, *args: Any, **kwargs: Any) -> Any: - cls = scyjava.jimport(full_name) + cls = scyjava.jimport(class_path) return cls(*args, **kwargs) Proxy.__name__ = name Proxy.__qualname__ = name Proxy.__module__ = module_name - Proxy.__doc__ = f"Proxy for {full_name}" + Proxy.__doc__ = f"Proxy for {class_path}" return Proxy return module_all, module_getattr diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 22c0eca..754b33f 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -116,6 +116,7 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: output_dir = Path(output_dir) if add_runtime_imports: logger.info("Adding runtime imports to generated stubs") + for stub in output_dir.rglob("*.pyi"): stub_ast = ast.parse(stub.read_text()) members = {node.name for node in stub_ast.body if hasattr(node, "name")} @@ -127,8 +128,13 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: continue if add_runtime_imports: real_import = stub.with_suffix(".py") - endpoint_args = ", ".join(repr(x) for x in endpoints) - real_import.write_text(INIT_TEMPLATE.format(endpoints=endpoint_args)) + base_prefix = stub.relative_to(output_dir).parts[0] + real_import.write_text( + INIT_TEMPLATE.format( + endpoints=repr(endpoints), + base_prefix=repr(base_prefix), + ) + ) ruff_check(output_dir.absolute()) @@ -141,7 +147,12 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: # see scyjava._stubs for implementation details. from scyjava._stubs import dynamic_import -__all__, __getattr__ = dynamic_import(__name__, __file__, {endpoints}) +__all__, __getattr__ = dynamic_import( + __name__, + __file__, + endpoints={endpoints}, + base_prefix={base_prefix}, +) """ From 2cb4836c370a678b307849891300d28a9777b6e1 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 23 Apr 2025 08:43:06 -0400 Subject: [PATCH 10/22] refactor: rename dynamic_import to setup_java_imports and update usage in stubs --- src/scyjava/_stubs/__init__.py | 4 +-- src/scyjava/_stubs/_dynamic_import.py | 41 ++++++++++++++++++++++++++- src/scyjava/_stubs/_genstubs.py | 13 +++++++-- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/scyjava/_stubs/__init__.py b/src/scyjava/_stubs/__init__.py index 4f42a0b..d6a5e7c 100644 --- a/src/scyjava/_stubs/__init__.py +++ b/src/scyjava/_stubs/__init__.py @@ -1,4 +1,4 @@ -from ._dynamic_import import dynamic_import +from ._dynamic_import import setup_java_imports from ._genstubs import generate_stubs -__all__ = ["dynamic_import", "generate_stubs"] +__all__ = ["setup_java_imports", "generate_stubs"] diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py index f62407b..3ce6669 100644 --- a/src/scyjava/_stubs/_dynamic_import.py +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -4,12 +4,51 @@ from typing import Any, Callable, Sequence -def dynamic_import( +def setup_java_imports( module_name: str, module_file: str, endpoints: Sequence[str] = (), base_prefix: str = "", ) -> tuple[list[str], Callable[[str], Any]]: + """Setup a module to dynamically import Java class names. + + This function creates a `__getattr__` function that, when called, will dynamically + import the requested class from the Java namespace corresponding to this module. + + :param module_name: The dotted name/identifier of the module that is calling this + function (usually `__name__` in the calling module). + :param module_file: The path to the module file (usually `__file__` in the calling + module). + :param endpoints: A list of Java endpoints to add to the scyjava configuration. + :param base_prefix: The base prefix for the Java package name. This is used when + determining the Java class path for the requested class. The java class path + will be truncated to only the part including the base_prefix and after. This + makes it possible to embed a module in a subpackage (like `scyjava.types`) and + still have the correct Java class path. + :return: A 2-tuple containing: + - A list of all classes in the module (as defined in the stub file), to be + assigned to `__all__`. + - A callable that takes a class name and returns a proxy for the Java class. + This callable should be assigned to `__getattr__` in the calling module. + The proxy object, when called, will start the JVM, import the Java class, + and return an instance of the class. The JVM will *only* be started when + the object is called. + + Example: + If the module calling this function is named `scyjava.types.org.scijava.parsington`, + then it should invoke this function as: + + .. code-block:: python + + from scyjava._stubs import setup_java_imports + + __all__, __getattr__ = setup_java_imports( + __name__, + __file__, + endpoints=["org.scijava:parsington:3.1.0"], + base_prefix="org" + ) + """ import scyjava import scyjava.config diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 754b33f..cb6bef2 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -15,7 +15,6 @@ import scyjava import scyjava.config -import stubgenj if TYPE_CHECKING: from collections.abc import Sequence @@ -69,6 +68,14 @@ def generate_stubs( the `__init__.pyi` for any given module will be whatever whatever the *last* stub generator wrote to it (and therefore inaccurate). """ + try: + import stubgenj + except ImportError as e: + raise ImportError( + "stubgenj is not installed, but is required to generate java stubs. " + "Please install it with `pip/conda install stubgenj`." + ) from e + import jpype startJVM = jpype.startJVM @@ -145,9 +152,9 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: # it creates a __getattr__ function that will dynamically import # the requested class from the Java namespace corresponding to this module. # see scyjava._stubs for implementation details. -from scyjava._stubs import dynamic_import +from scyjava._stubs import setup_java_imports -__all__, __getattr__ = dynamic_import( +__all__, __getattr__ = setup_java_imports( __name__, __file__, endpoints={endpoints}, From 71f761ed3972b484ae064e73354336696412b101 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 23 Apr 2025 08:43:58 -0400 Subject: [PATCH 11/22] reword --- src/scyjava/_stubs/_dynamic_import.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py index 3ce6669..cc62641 100644 --- a/src/scyjava/_stubs/_dynamic_import.py +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -13,7 +13,8 @@ def setup_java_imports( """Setup a module to dynamically import Java class names. This function creates a `__getattr__` function that, when called, will dynamically - import the requested class from the Java namespace corresponding to this module. + import the requested class from the Java namespace corresponding to the calling + module. :param module_name: The dotted name/identifier of the module that is calling this function (usually `__name__` in the calling module). From 65cc471a5df976c61f275a5bf42a7abd066b168b Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 24 Apr 2025 21:26:03 -0400 Subject: [PATCH 12/22] feat: add Hatchling build hook for generating Java stubs --- pyproject.toml | 13 +++++------ src/scyjava/_stubs/_hatchling.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 src/scyjava/_stubs/_hatchling.py diff --git a/pyproject.toml b/pyproject.toml index b36f2bf..0d7fb64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,12 +32,7 @@ classifiers = [ # NB: Keep this in sync with environment.yml AND dev-environment.yml! requires-python = ">=3.9" -dependencies = [ - "jpype1 >= 1.3.0", - "jgo", - "cjdk", - "stubgenj", -] +dependencies = ["jpype1 >= 1.3.0", "jgo", "cjdk", "stubgenj"] [project.optional-dependencies] # NB: Keep this in sync with dev-environment.yml! @@ -51,12 +46,16 @@ dev = [ "pandas", "ruff", "toml", - "validate-pyproject[all]" + "validate-pyproject[all]", ] [project.scripts] scyjava-stubgen = "scyjava._stubs._cli:main" +[project.entry-points.hatch] +mypyc = "scyjava._stubs._hatchling" + + [project.urls] homepage = "https://github.com/scijava/scyjava" documentation = "https://github.com/scijava/scyjava/blob/main/README.md" diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling.py new file mode 100644 index 0000000..d4c1ade --- /dev/null +++ b/src/scyjava/_stubs/_hatchling.py @@ -0,0 +1,38 @@ +"""Hatchling build hook for generating Java stubs.""" + +import shutil +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.plugin import hookimpl + + +from scyjava._stubs._genstubs import generate_stubs + + +class ScyjavaBuildHook(BuildHookInterface): + """Custom build hook for generating Java stubs.""" + + PLUGIN_NAME = "scyjava" + + def initialize(self, version: str, build_data: dict) -> None: + """Initialize the build hook with the version and build data.""" + breakpoint() + if self.target_name != "wheel": + return + dest = Path(self.root, "src") + shutil.rmtree(dest, ignore_errors=True) # remove the old stubs + + # actually build the stubs + coord = f"{self.config['maven_coord']}:{self.metadata.version}" + prefixes = self.config.get("prefixes", []) + generate_stubs(endpoints=[coord], prefixes=prefixes, output_dir=dest) + + # add all packages to the build config + packages = [str(x.relative_to(self.root)) for x in dest.iterdir()] + self.build_config.target_config.setdefault("packages", packages) + + +@hookimpl +def hatch_register_build_hook(): + return ScyjavaBuildHook From 6e4181e77c360920545651ebb1b2b652cc02db52 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 24 Apr 2025 21:54:57 -0400 Subject: [PATCH 13/22] wip --- src/scyjava/_jvm.py | 2 +- src/scyjava/_stubs/_hatchling.py | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/scyjava/_jvm.py b/src/scyjava/_jvm.py index 2035e34..d8a5bed 100644 --- a/src/scyjava/_jvm.py +++ b/src/scyjava/_jvm.py @@ -373,7 +373,7 @@ def is_awt_initialized() -> bool: return False Thread = scyjava.jimport("java.lang.Thread") threads = Thread.getAllStackTraces().keySet() - return any(t.getName().startsWith("AWT-") for t in threads) + return any(str(t.getName()).startswith("AWT-") for t in threads) def when_jvm_starts(f) -> None: diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling.py index d4c1ade..140b901 100644 --- a/src/scyjava/_stubs/_hatchling.py +++ b/src/scyjava/_stubs/_hatchling.py @@ -1,5 +1,6 @@ """Hatchling build hook for generating Java stubs.""" +import logging import shutil from pathlib import Path @@ -9,6 +10,8 @@ from scyjava._stubs._genstubs import generate_stubs +logger = logging.getLogger("scyjava") + class ScyjavaBuildHook(BuildHookInterface): """Custom build hook for generating Java stubs.""" @@ -17,20 +20,22 @@ class ScyjavaBuildHook(BuildHookInterface): def initialize(self, version: str, build_data: dict) -> None: """Initialize the build hook with the version and build data.""" - breakpoint() if self.target_name != "wheel": return - dest = Path(self.root, "src") - shutil.rmtree(dest, ignore_errors=True) # remove the old stubs - # actually build the stubs - coord = f"{self.config['maven_coord']}:{self.metadata.version}" + endpoints = self.config.get("maven_coordinates", []) + if not endpoints: + logger.warning("No maven coordinates provided. Skipping stub generation.") + return + prefixes = self.config.get("prefixes", []) - generate_stubs(endpoints=[coord], prefixes=prefixes, output_dir=dest) + dest = Path(self.root, "src", "scyjava", "types") - # add all packages to the build config - packages = [str(x.relative_to(self.root)) for x in dest.iterdir()] - self.build_config.target_config.setdefault("packages", packages) + # actually build the stubs + generate_stubs(endpoints=endpoints, prefixes=prefixes, output_dir=dest) + print(f"Generated stubs for {endpoints} in {dest}") + # add all new packages to the build config + build_data["artifacts"].append("src/scyjava/types") @hookimpl From 6e92b13720484ee55f33b92c8c9929ff5c52dbef Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 25 Apr 2025 08:05:33 -0400 Subject: [PATCH 14/22] fix inclusion --- src/scyjava/_stubs/_hatchling.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling.py index 140b901..52b18e2 100644 --- a/src/scyjava/_stubs/_hatchling.py +++ b/src/scyjava/_stubs/_hatchling.py @@ -30,12 +30,18 @@ def initialize(self, version: str, build_data: dict) -> None: prefixes = self.config.get("prefixes", []) dest = Path(self.root, "src", "scyjava", "types") + (dest.parent / "py.typed").touch() # actually build the stubs - generate_stubs(endpoints=endpoints, prefixes=prefixes, output_dir=dest) + generate_stubs( + endpoints=endpoints, + prefixes=prefixes, + output_dir=dest, + remove_namespace_only_stubs=True, + ) print(f"Generated stubs for {endpoints} in {dest}") # add all new packages to the build config - build_data["artifacts"].append("src/scyjava/types") + build_data["force_include"].update({str(dest.parent): "scyjava"}) @hookimpl From 0d231cccf93dc634bda853647252040063f66cdd Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 25 Apr 2025 08:08:06 -0400 Subject: [PATCH 15/22] add docs --- src/scyjava/_stubs/_hatchling.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling.py index 52b18e2..866f829 100644 --- a/src/scyjava/_stubs/_hatchling.py +++ b/src/scyjava/_stubs/_hatchling.py @@ -1,7 +1,23 @@ -"""Hatchling build hook for generating Java stubs.""" +"""Hatchling build hook for generating Java stubs. + +To use this hook, add the following to your `pyproject.toml`: + +```toml +[build-system] +requires = ["hatchling", "scyjava"] +build-backend = "hatchling.build" + +[tool.hatch.build.hooks.scyjava] +maven_coordinates = ["org.scijava:parsington:3.1.0"] +prefixes = ["org.scijava"] # optional ... can be auto-determined from the jar files +``` + +This will generate stubs for the given maven coordinates and prefixes. The generated +stubs will be placed in `src/scyjava/types` and will be included in the wheel package. +This hook is only run when building a wheel package. +""" import logging -import shutil from pathlib import Path from hatchling.builders.hooks.plugin.interface import BuildHookInterface From 79524b0065c0e6bdbb57c63d9d2cbe9bc8cc6c8b Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 25 Apr 2025 08:17:38 -0400 Subject: [PATCH 16/22] setuptools plugin stub --- pyproject.toml | 5 +- .../{_hatchling.py => _hatchling_plugin.py} | 0 src/scyjava/_stubs/_setuptools_plugin.py | 79 +++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) rename src/scyjava/_stubs/{_hatchling.py => _hatchling_plugin.py} (100%) create mode 100644 src/scyjava/_stubs/_setuptools_plugin.py diff --git a/pyproject.toml b/pyproject.toml index 0d7fb64..47f3ffb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,8 +53,9 @@ dev = [ scyjava-stubgen = "scyjava._stubs._cli:main" [project.entry-points.hatch] -mypyc = "scyjava._stubs._hatchling" - +scyjava = "scyjava._stubs._hatchling_plugin" +[project.entry-points."distutils.commands"] +build_py = "scyjava_stubgen.build:build_py" [project.urls] homepage = "https://github.com/scijava/scyjava" diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling_plugin.py similarity index 100% rename from src/scyjava/_stubs/_hatchling.py rename to src/scyjava/_stubs/_hatchling_plugin.py diff --git a/src/scyjava/_stubs/_setuptools_plugin.py b/src/scyjava/_stubs/_setuptools_plugin.py new file mode 100644 index 0000000..e9386f9 --- /dev/null +++ b/src/scyjava/_stubs/_setuptools_plugin.py @@ -0,0 +1,79 @@ +"""Setuptools build hook for generating Java stubs. + +To use this hook, add the following to your `pyproject.toml`: + +```toml +[build-system] +requires = ["setuptools>=69", "wheel", "scyjava"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.cmdclass] +build_py = "scyjava_stubgen.build:build_py" +# optional project-specific defaults +maven_coordinates = ["org.scijava:parsington:3.1.0"] +prefixes = ["org.scijava"] +``` + +This will generate stubs for the given maven coordinates and prefixes. The generated +stubs will be placed in `src/scyjava/types` and will be included in the wheel package. +This hook is only run when building a wheel package. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import List + +from setuptools.command.build_py import build_py as _build_py +from scyjava._stubs._genstubs import generate_stubs + +log = logging.getLogger("scyjava") + + +class build_py(_build_py): # type: ignore[misc] + """ + A drop-in replacement for setuptools' build_py that + generates Java type stubs before Python sources are copied + into *build_lib*. + """ + + # expose two optional CLI/pyproject options so users can override defaults + user_options: List[tuple[str, str | None, str]] = _build_py.user_options + [ + ("maven-coordinates=", None, "List of Maven coordinates to stub"), + ("prefixes=", None, "Java package prefixes to include"), + ] + + def initialize_options(self) -> None: # noqa: D401 + super().initialize_options() + self.maven_coordinates: list[str] | None = None + self.prefixes: list[str] | None = None + + def finalize_options(self) -> None: # noqa: D401 + """Fill in options that may come from pyproject metadata.""" + super().finalize_options() + dist = self.distribution # alias + if self.maven_coordinates is None: + self.maven_coordinates = getattr(dist, "maven_coordinates", []) + if self.prefixes is None: + self.prefixes = getattr(dist, "prefixes", []) + + def run(self) -> None: # noqa: D401 + """Generate stubs, then let the normal build_py proceed.""" + if self.maven_coordinates: + dest = Path(self.build_lib, "scyjava", "types") + dest.parent.mkdir(parents=True, exist_ok=True) + (dest.parent / "py.typed").touch() + + generate_stubs( + endpoints=self.maven_coordinates, + prefixes=self.prefixes, + output_dir=dest, + remove_namespace_only_stubs=True, + ) + log.info("Generated stubs for %s", ", ".join(self.maven_coordinates)) + + # make sure the wheel knows about them + self.package_data.setdefault("scyjava", []).append("types/**/*.pyi") + + super().run() From 51937b587562d13922b4dfd944fd142c5c5fee6a Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 30 Apr 2025 20:17:11 -0400 Subject: [PATCH 17/22] remove repr test --- tests/test_stubgen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py index 300ee7f..0d4c28d 100644 --- a/tests/test_stubgen.py +++ b/tests/test_stubgen.py @@ -46,7 +46,6 @@ def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: from org.scijava.parsington import Function assert Function is not None - assert repr(Function) == "" # ensure that no calls to start_jvm were made mock_start_jvm.assert_not_called() From 192be35b1c7ad073064589a348af4be96b65723e Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 1 May 2025 09:05:49 -0400 Subject: [PATCH 18/22] skip in jep --- tests/test_stubgen.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py index 0d4c28d..ad4a74a 100644 --- a/tests/test_stubgen.py +++ b/tests/test_stubgen.py @@ -5,6 +5,7 @@ from unittest.mock import patch import jpype +import pytest import scyjava from scyjava._stubs import _cli @@ -12,9 +13,11 @@ if TYPE_CHECKING: from pathlib import Path - import pytest - +@pytest.mark.skipif( + scyjava.config.mode != scyjava.config.Mode.JPYPE, + reason="Stubgen not supported in JEP", +) def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: # run the stubgen command as if it was run from the command line monkeypatch.setattr( From e714abb7a00df87550fd3fee13faaeb55593ce64 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 1 May 2025 09:12:41 -0400 Subject: [PATCH 19/22] remove setuptools plugin --- src/scyjava/_stubs/_setuptools_plugin.py | 79 ------------------------ 1 file changed, 79 deletions(-) delete mode 100644 src/scyjava/_stubs/_setuptools_plugin.py diff --git a/src/scyjava/_stubs/_setuptools_plugin.py b/src/scyjava/_stubs/_setuptools_plugin.py deleted file mode 100644 index e9386f9..0000000 --- a/src/scyjava/_stubs/_setuptools_plugin.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Setuptools build hook for generating Java stubs. - -To use this hook, add the following to your `pyproject.toml`: - -```toml -[build-system] -requires = ["setuptools>=69", "wheel", "scyjava"] -build-backend = "setuptools.build_meta" - -[tool.setuptools.cmdclass] -build_py = "scyjava_stubgen.build:build_py" -# optional project-specific defaults -maven_coordinates = ["org.scijava:parsington:3.1.0"] -prefixes = ["org.scijava"] -``` - -This will generate stubs for the given maven coordinates and prefixes. The generated -stubs will be placed in `src/scyjava/types` and will be included in the wheel package. -This hook is only run when building a wheel package. -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import List - -from setuptools.command.build_py import build_py as _build_py -from scyjava._stubs._genstubs import generate_stubs - -log = logging.getLogger("scyjava") - - -class build_py(_build_py): # type: ignore[misc] - """ - A drop-in replacement for setuptools' build_py that - generates Java type stubs before Python sources are copied - into *build_lib*. - """ - - # expose two optional CLI/pyproject options so users can override defaults - user_options: List[tuple[str, str | None, str]] = _build_py.user_options + [ - ("maven-coordinates=", None, "List of Maven coordinates to stub"), - ("prefixes=", None, "Java package prefixes to include"), - ] - - def initialize_options(self) -> None: # noqa: D401 - super().initialize_options() - self.maven_coordinates: list[str] | None = None - self.prefixes: list[str] | None = None - - def finalize_options(self) -> None: # noqa: D401 - """Fill in options that may come from pyproject metadata.""" - super().finalize_options() - dist = self.distribution # alias - if self.maven_coordinates is None: - self.maven_coordinates = getattr(dist, "maven_coordinates", []) - if self.prefixes is None: - self.prefixes = getattr(dist, "prefixes", []) - - def run(self) -> None: # noqa: D401 - """Generate stubs, then let the normal build_py proceed.""" - if self.maven_coordinates: - dest = Path(self.build_lib, "scyjava", "types") - dest.parent.mkdir(parents=True, exist_ok=True) - (dest.parent / "py.typed").touch() - - generate_stubs( - endpoints=self.maven_coordinates, - prefixes=self.prefixes, - output_dir=dest, - remove_namespace_only_stubs=True, - ) - log.info("Generated stubs for %s", ", ".join(self.maven_coordinates)) - - # make sure the wheel knows about them - self.package_data.setdefault("scyjava", []).append("types/**/*.pyi") - - super().run() From a18d9d908e2e11263679eb09ae3b3b14d17553ce Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 22 Aug 2025 19:17:00 -0400 Subject: [PATCH 20/22] remove hatch plugin --- pyproject.toml | 5 -- src/scyjava/_stubs/_genstubs.py | 2 +- src/scyjava/_stubs/_hatchling_plugin.py | 65 ------------------------- 3 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 src/scyjava/_stubs/_hatchling_plugin.py diff --git a/pyproject.toml b/pyproject.toml index 3471c09..76c8882 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,11 +49,6 @@ dev = [ [project.scripts] scyjava-stubgen = "scyjava._stubs._cli:main" -[project.entry-points.hatch] -scyjava = "scyjava._stubs._hatchling_plugin" -[project.entry-points."distutils.commands"] -build_py = "scyjava_stubgen.build:build_py" - [project.urls] homepage = "https://github.com/scijava/scyjava" documentation = "https://github.com/scijava/scyjava/blob/main/README.md" diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index cb6bef2..1576d64 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -5,10 +5,10 @@ import os import shutil import subprocess +import sys from importlib import import_module from itertools import chain from pathlib import Path, PurePath -import sys from typing import TYPE_CHECKING, Any from unittest.mock import patch from zipfile import ZipFile diff --git a/src/scyjava/_stubs/_hatchling_plugin.py b/src/scyjava/_stubs/_hatchling_plugin.py deleted file mode 100644 index 866f829..0000000 --- a/src/scyjava/_stubs/_hatchling_plugin.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Hatchling build hook for generating Java stubs. - -To use this hook, add the following to your `pyproject.toml`: - -```toml -[build-system] -requires = ["hatchling", "scyjava"] -build-backend = "hatchling.build" - -[tool.hatch.build.hooks.scyjava] -maven_coordinates = ["org.scijava:parsington:3.1.0"] -prefixes = ["org.scijava"] # optional ... can be auto-determined from the jar files -``` - -This will generate stubs for the given maven coordinates and prefixes. The generated -stubs will be placed in `src/scyjava/types` and will be included in the wheel package. -This hook is only run when building a wheel package. -""" - -import logging -from pathlib import Path - -from hatchling.builders.hooks.plugin.interface import BuildHookInterface -from hatchling.plugin import hookimpl - - -from scyjava._stubs._genstubs import generate_stubs - -logger = logging.getLogger("scyjava") - - -class ScyjavaBuildHook(BuildHookInterface): - """Custom build hook for generating Java stubs.""" - - PLUGIN_NAME = "scyjava" - - def initialize(self, version: str, build_data: dict) -> None: - """Initialize the build hook with the version and build data.""" - if self.target_name != "wheel": - return - - endpoints = self.config.get("maven_coordinates", []) - if not endpoints: - logger.warning("No maven coordinates provided. Skipping stub generation.") - return - - prefixes = self.config.get("prefixes", []) - dest = Path(self.root, "src", "scyjava", "types") - (dest.parent / "py.typed").touch() - - # actually build the stubs - generate_stubs( - endpoints=endpoints, - prefixes=prefixes, - output_dir=dest, - remove_namespace_only_stubs=True, - ) - print(f"Generated stubs for {endpoints} in {dest}") - # add all new packages to the build config - build_data["force_include"].update({str(dest.parent): "scyjava"}) - - -@hookimpl -def hatch_register_build_hook(): - return ScyjavaBuildHook From b53e795b5e1dac8e3c2037ce91d41e06c59e3233 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 22 Aug 2025 19:18:02 -0400 Subject: [PATCH 21/22] newlines --- pyproject.toml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 76c8882..ccb3fb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "scyjava" version = "1.12.2.dev0" description = "Supercharged Java access from Python" license = "Unlicense" -authors = [{name = "SciJava developers", email = "ctrueden@wisc.edu"}] +authors = [{ name = "SciJava developers", email = "ctrueden@wisc.edu" }] readme = "README.md" keywords = ["java", "maven", "cross-language"] classifiers = [ @@ -31,7 +31,12 @@ classifiers = [ ] requires-python = ">=3.9" -dependencies = ["jpype1 >= 1.3.0", "jgo", "cjdk", "stubgenj"] +dependencies = [ + "jpype1 >= 1.3.0", + "jgo", + "cjdk", + "stubgenj", +] [dependency-groups] dev = [ @@ -57,7 +62,7 @@ download = "https://pypi.org/project/scyjava/" tracker = "https://github.com/scijava/scyjava/issues" [tool.setuptools] -package-dir = {"" = "src"} +package-dir = { "" = "src" } include-package-data = false [tool.setuptools.packages.find] From ed0add6b06a71dd729fe502c6c8ec16588e9e5e1 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 22 Aug 2025 19:35:07 -0400 Subject: [PATCH 22/22] update docs --- src/scyjava/_stubs/_cli.py | 10 ++++++++- src/scyjava/_stubs/_dynamic_import.py | 29 +++++++++++++++++++++++++++ src/scyjava/_stubs/_genstubs.py | 27 +++++++++++++++++++------ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/scyjava/_stubs/_cli.py b/src/scyjava/_stubs/_cli.py index 3a4d7df..7936529 100644 --- a/src/scyjava/_stubs/_cli.py +++ b/src/scyjava/_stubs/_cli.py @@ -1,4 +1,12 @@ -"""The scyjava-stubs executable.""" +"""The scyjava-stubs executable. + +Provides cli access to the `scyjava._stubs.generate_stubs` function. + +The only interesting additional things going on here is the choice of *where* the stubs +go by default. When using the CLI, they land in `scyjava.types` by default; see the +`_get_ouput_dir` helper function for details on how the output directory is resolved +from the CLI arguments. +""" from __future__ import annotations diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py index cc62641..16e27d4 100644 --- a/src/scyjava/_stubs/_dynamic_import.py +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -1,3 +1,24 @@ +"""Logic for using generated type stubs as runtime importable, with lazy JVM startup. + +Most often, the functionality here will be used as follows: + +``` +from scyjava._stubs import setup_java_imports + +__all__, __getattr__ = setup_java_imports( + __name__, + __file__, + endpoints=["org.scijava:parsington:3.1.0"], + base_prefix="org" +) +``` + +...and that little snippet is written into the generated stubs modules by the +`scyjava._stubs.generate_stubs` function. + +See docstring of `setup_java_imports` for details on how it works. +""" + import ast from logging import warning from pathlib import Path @@ -21,11 +42,14 @@ def setup_java_imports( :param module_file: The path to the module file (usually `__file__` in the calling module). :param endpoints: A list of Java endpoints to add to the scyjava configuration. + (Note that `scyjava._stubs.generate_stubs` will automatically add the necessary + endpoints for the generated stubs.) :param base_prefix: The base prefix for the Java package name. This is used when determining the Java class path for the requested class. The java class path will be truncated to only the part including the base_prefix and after. This makes it possible to embed a module in a subpackage (like `scyjava.types`) and still have the correct Java class path. + :return: A 2-tuple containing: - A list of all classes in the module (as defined in the stub file), to be assigned to `__all__`. @@ -57,6 +81,7 @@ def setup_java_imports( if ep not in scyjava.config.endpoints: scyjava.config.endpoints.append(ep) + # list intended to be assigned to `__all__` in the generated module. module_all = [] try: my_stub = Path(module_file).with_suffix(".pyi") @@ -75,6 +100,7 @@ def setup_java_imports( ) def module_getattr(name: str, mod_name: str = module_name) -> Any: + """Function intended to be assigned to __getattr__ in the generate module.""" if module_all and name not in module_all: raise AttributeError(f"module {module_name!r} has no attribute {name!r}") @@ -84,6 +110,9 @@ def module_getattr(name: str, mod_name: str = module_name) -> Any: class_path = f"{mod_name}.{name}" + # Generate a proxy type (with a nice repr) that + # delays the call to `jimport` until the last moment when type.__new__ is called + class ProxyMeta(type): def __repr__(self) -> str: return f"" diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 1576d64..3df24cb 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -1,3 +1,14 @@ +"""Type stub generation utilities using stubgen. + +This module provides utilities for generating type stubs for Java classes +using the stubgenj library. `stubgenj` must be installed for this to work +(it, in turn, only depends on JPype). + +See `generate_stubs` for most functionality. For the command-line tool, +see `scyjava._stubs.cli`, which provides a CLI interface for the `generate_stubs` +function. +""" + from __future__ import annotations import ast @@ -42,9 +53,11 @@ def generate_stubs( The prefixes to generate stubs for. This should be a list of Java class prefixes that you expect to find in the endpoints. For example, ["org.apache.commons"]. If not provided, the prefixes will be - automatically determined from the jar files provided by endpoints. + automatically determined from the jar files provided by endpoints (see the + `_list_top_level_packages` helper function). output_dir : str | Path, optional - The directory to write the generated stubs to. Defaults to "stubs". + The directory to write the generated stubs to. Defaults to "stubs" in the + current working directory. convert_strings : bool, optional Whether to cast Java strings to Python strings in the stubs. Defaults to True. NOTE: This leads to type stubs that may not be strictly accurate at runtime. @@ -58,8 +71,10 @@ def generate_stubs( Whether to include Javadoc in the generated stubs. Defaults to True. add_runtime_imports : bool, optional Whether to add runtime imports to the generated stubs. Defaults to True. - This is useful if you want to use the stubs as a runtime package with type - safety. + This is useful if you want to actually import the stubs as a runtime package + with type safety. The runtime import "magic" depends on the + `scyjava._stubs.setup_java_imports` function. See its documentation for + more details. remove_namespace_only_stubs : bool, optional Whether to remove stubs that export no names beyond a single `__module_protocol__`. This leaves some folders as PEP420 implicit namespace @@ -95,7 +110,7 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: ep_artifacts = tuple(ep.split(":")[1] for ep in endpoints) for j in cp.split(os.pathsep): if Path(j).name.startswith(ep_artifacts): - _prefixes.update(list_top_level_packages(j)) + _prefixes.update(_list_top_level_packages(j)) prefixes = sorted(_prefixes) logger.info(f"Using endpoints: {scyjava.config.endpoints!r}") @@ -189,7 +204,7 @@ def ruff_check(output: Path, select: str = "E,W,F,I,UP,C4,B,RUF,TC,TID") -> None subprocess.run(["ruff", "format", *py_files, "--quiet"]) -def list_top_level_packages(jar_path: str) -> set[str]: +def _list_top_level_packages(jar_path: str) -> set[str]: """Inspect a JAR file and return the set of top-level Java package names.""" packages: set[str] = set() with ZipFile(jar_path, "r") as jar: