From 0f70b037aff5284b15601d5683b861a0df7496d1 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 22:50:48 -0400 Subject: [PATCH 01/29] ignore types folder --- src/scyjava/types/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/scyjava/types/.gitignore diff --git a/src/scyjava/types/.gitignore b/src/scyjava/types/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/src/scyjava/types/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore From 459a9992539a193b11ceccb625e938218a26b7eb Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:17:04 -0400 Subject: [PATCH 02/29] Add scyjava-stubs CLI and dynamic import functionality - Introduced `scyjava-stubs` executable for generating Python type stubs from Java classes. - Implemented dynamic import logic in `_dynamic_import.py`. - Added stub generation logic in `_genstubs.py`. - Updated `pyproject.toml` to include new dependencies and scripts. - Created `__init__.py` for the `_stubs` package to expose key functionalities. --- pyproject.toml | 4 + src/scyjava/_stubs/__init__.py | 4 + src/scyjava/_stubs/_cli.py | 152 +++++++++++++++++++++ src/scyjava/_stubs/_dynamic_import.py | 61 +++++++++ src/scyjava/_stubs/_genstubs.py | 184 ++++++++++++++++++++++++++ 5 files changed, 405 insertions(+) create mode 100644 src/scyjava/_stubs/__init__.py create mode 100644 src/scyjava/_stubs/_cli.py create mode 100644 src/scyjava/_stubs/_dynamic_import.py create mode 100644 src/scyjava/_stubs/_genstubs.py diff --git a/pyproject.toml b/pyproject.toml index 8da9830..b36f2bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "jpype1 >= 1.3.0", "jgo", "cjdk", + "stubgenj", ] [project.optional-dependencies] @@ -53,6 +54,9 @@ dev = [ "validate-pyproject[all]" ] +[project.scripts] +scyjava-stubgen = "scyjava._stubs._cli:main" + [project.urls] homepage = "https://github.com/scijava/scyjava" documentation = "https://github.com/scijava/scyjava/blob/main/README.md" diff --git a/src/scyjava/_stubs/__init__.py b/src/scyjava/_stubs/__init__.py new file mode 100644 index 0000000..4f42a0b --- /dev/null +++ b/src/scyjava/_stubs/__init__.py @@ -0,0 +1,4 @@ +from ._dynamic_import import dynamic_import +from ._genstubs import generate_stubs + +__all__ = ["dynamic_import", "generate_stubs"] diff --git a/src/scyjava/_stubs/_cli.py b/src/scyjava/_stubs/_cli.py new file mode 100644 index 0000000..eeb6970 --- /dev/null +++ b/src/scyjava/_stubs/_cli.py @@ -0,0 +1,152 @@ +"""The scyjava-stubs executable.""" + +import argparse +import importlib +import importlib.util +import logging +import sys +from pathlib import Path + +from ._genstubs import generate_stubs + + +def main() -> None: + """The main entry point for the scyjava-stubs executable.""" + logging.basicConfig(level="INFO") + parser = argparse.ArgumentParser( + description="Generate Python Type Stubs for Java classes." + ) + parser.add_argument( + "endpoints", + type=str, + nargs="+", + help="Maven endpoints to install and use (e.g. org.myproject:myproject:1.0.0)", + ) + parser.add_argument( + "--prefix", + type=str, + help="package prefixes to generate stubs for (e.g. org.myproject), " + "may be used multiple times. If not specified, prefixes are gleaned from the " + "downloaded artifacts.", + action="append", + default=[], + metavar="PREFIX", + dest="prefix", + ) + path_group = parser.add_mutually_exclusive_group() + path_group.add_argument( + "--output-dir", + type=str, + default=None, + help="Filesystem path to write stubs to.", + ) + path_group.add_argument( + "--output-python-path", + type=str, + default=None, + help="Python path to write stubs to (e.g. 'scyjava.types').", + ) + parser.add_argument( + "--convert-strings", + dest="convert_strings", + action="store_true", + default=False, + help="convert java.lang.String to python str in return types. " + "consult the JPype documentation on the convertStrings flag for details", + ) + parser.add_argument( + "--no-javadoc", + dest="with_javadoc", + action="store_false", + default=True, + help="do not generate docstrings from JavaDoc where available", + ) + + rt_group = parser.add_mutually_exclusive_group() + rt_group.add_argument( + "--runtime-imports", + dest="runtime_imports", + action="store_true", + default=True, + help="Add runtime imports to the generated stubs. ", + ) + rt_group.add_argument( + "--no-runtime-imports", dest="runtime_imports", action="store_false" + ) + + parser.add_argument( + "--remove-namespace-only-stubs", + dest="remove_namespace_only_stubs", + action="store_true", + default=False, + help="Remove stubs that export no names beyond a single __module_protocol__. " + "This leaves some folders as PEP420 implicit namespace folders.", + ) + + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + + args = parser.parse_args() + output_dir = _get_ouput_dir(args.output_dir, args.output_python_path) + if not output_dir.exists(): + output_dir.mkdir(parents=True, exist_ok=True) + + generate_stubs( + endpoints=args.endpoints, + prefixes=args.prefix, + output_dir=output_dir, + convert_strings=args.convert_strings, + include_javadoc=args.with_javadoc, + add_runtime_imports=args.runtime_imports, + remove_namespace_only_stubs=args.remove_namespace_only_stubs, + ) + + +def _get_ouput_dir(output_dir: str | None, python_path: str | None) -> Path: + if out_dir := output_dir: + return Path(out_dir) + if pp := python_path: + return _glean_path(pp) + try: + import scyjava + + return Path(scyjava.__file__).parent / "types" + except ImportError: + return Path("stubs") + + +def _glean_path(pp: str) -> Path: + try: + importlib.import_module(pp.split(".")[0]) + except ModuleNotFoundError: + # the top level module doesn't exist: + raise ValueError(f"Module {pp} does not exist. Cannot install stubs there.") + + try: + spec = importlib.util.find_spec(pp) + except ModuleNotFoundError as e: + # at least one of the middle levels doesn't exist: + raise NotImplementedError(f"Cannot install stubs to {pp}: {e}") + + new_ns = None + if not spec: + # if we get here, it means everything but the last level exists: + parent, new_ns = pp.rsplit(".", 1) + spec = importlib.util.find_spec(parent) + + if not spec: + # if we get here, it means the last level doesn't exist: + raise ValueError(f"Module {pp} does not exist. Cannot install stubs there.") + + search_locations = spec.submodule_search_locations + if not spec.loader and search_locations: + # namespace package with submodules + return Path(search_locations[0]) + if spec.origin: + return Path(spec.origin).parent + if new_ns and search_locations: + # namespace package with submodules + return Path(search_locations[0]) / new_ns + + raise ValueError(f"Error finding module {pp}. Cannot install stubs there.") diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py new file mode 100644 index 0000000..af50065 --- /dev/null +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -0,0 +1,61 @@ +import ast +from logging import warning +from pathlib import Path +from typing import Any, Callable + + +def dynamic_import( + module_name: str, module_file: str, *endpoints: str +) -> tuple[list[str], Callable[[str], Any]]: + import scyjava + import scyjava.config + + for ep in endpoints: + if ep not in scyjava.config.endpoints: + scyjava.config.endpoints.append(ep) + + module_all = [] + try: + my_stub = Path(module_file).with_suffix(".pyi") + stub_ast = ast.parse(my_stub.read_text()) + module_all = sorted( + { + node.name + for node in stub_ast.body + if isinstance(node, ast.ClassDef) and not node.name.startswith("__") + } + ) + except (OSError, SyntaxError): + warning( + f"Failed to read stub file {my_stub!r}. Falling back to empty __all__.", + stacklevel=3, + ) + + def module_getattr(name: str, mod_name: str = module_name) -> Any: + if module_all and name not in module_all: + raise AttributeError(f"module {module_name!r} has no attribute {name!r}") + + # this strip is important... and tricky, because it depends on the + # namespace that we intend to install the stubs into. + install_path = "scyjava.types." + if mod_name.startswith(install_path): + mod_name = mod_name[len(install_path) :] + + full_name = f"{mod_name}.{name}" + + class ProxyMeta(type): + def __repr__(self) -> str: + return f"" + + class Proxy(metaclass=ProxyMeta): + def __new__(_cls_, *args: Any, **kwargs: Any) -> Any: + cls = scyjava.jimport(full_name) + return cls(*args, **kwargs) + + Proxy.__name__ = name + Proxy.__qualname__ = name + Proxy.__module__ = module_name + Proxy.__doc__ = f"Proxy for {full_name}" + return Proxy + + return module_all, module_getattr diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py new file mode 100644 index 0000000..be800bd --- /dev/null +++ b/src/scyjava/_stubs/_genstubs.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import ast +import logging +import os +import shutil +import subprocess +from importlib import import_module +from itertools import chain +from pathlib import Path, PurePath +from typing import TYPE_CHECKING, Any +from unittest.mock import patch +from zipfile import ZipFile + +import cjdk +import scyjava +import scyjava.config +import stubgenj + +if TYPE_CHECKING: + from collections.abc import Sequence + +logger = logging.getLogger(__name__) + + +def generate_stubs( + endpoints: Sequence[str], + prefixes: Sequence[str] = (), + output_dir: str | Path = "stubs", + convert_strings: bool = True, + include_javadoc: bool = True, + add_runtime_imports: bool = True, + remove_namespace_only_stubs: bool = False, +) -> None: + """Generate stubs for the given maven endpoints. + + Parameters + ---------- + endpoints : Sequence[str] + The maven endpoints to generate stubs for. This should be a list of GAV + coordinates, e.g. ["org.apache.commons:commons-lang3:3.12.0"]. + prefixes : Sequence[str], optional + The prefixes to generate stubs for. This should be a list of Java class + prefixes that you expect to find in the endpoints. For example, + ["org.apache.commons"]. If not provided, the prefixes will be + automatically determined from the jar files provided by endpoints. + output_dir : str | Path, optional + The directory to write the generated stubs to. Defaults to "stubs". + convert_strings : bool, optional + Whether to cast Java strings to Python strings in the stubs. Defaults to True. + NOTE: This leads to type stubs that may not be strictly accurate at runtime. + The actual runtime type of strings is determined by whether jpype.startJVM is + called with the `convertStrings` argument set to True or False. By setting + this `convert_strings` argument to true, the type stubs will be generated as if + `convertStrings` is set to True: that is, all string types will be listed as + `str` rather than `java.lang.String | str`. This is a safer default (as `str`) + is a subtype of `java.lang.String`), but may lead to type errors in some cases. + include_javadoc : bool, optional + Whether to include Javadoc in the generated stubs. Defaults to True. + add_runtime_imports : bool, optional + Whether to add runtime imports to the generated stubs. Defaults to True. + This is useful if you want to use the stubs as a runtime package with type + safety. + remove_namespace_only_stubs : bool, optional + Whether to remove stubs that export no names beyond a single + `__module_protocol__`. This leaves some folders as PEP420 implicit namespace + folders. Defaults to False. Setting this to `True` is useful if you want to + merge the generated stubs with other stubs in the same namespace. Without this, + the `__init__.pyi` for any given module will be whatever whatever the *last* + stub generator wrote to it (and therefore inaccurate). + """ + import jpype + import jpype.imports + + startJVM = jpype.startJVM + + scyjava.config.endpoints.extend(endpoints) + + def _patched_start(*args: Any, **kwargs: Any) -> None: + kwargs.setdefault("convertStrings", convert_strings) + startJVM(*args, **kwargs) + + with patch.object(jpype, "startJVM", new=_patched_start): + scyjava.start_jvm() + + _prefixes = set(prefixes) + if not _prefixes: + cp = jpype.getClassPath(env=False) + ep_artifacts = tuple(ep.split(":")[1] for ep in endpoints) + for j in cp.split(os.pathsep): + if Path(j).name.startswith(ep_artifacts): + _prefixes.update(list_top_level_packages(j)) + + prefixes = sorted(_prefixes) + logger.info(f"Using endpoints: {scyjava.config.endpoints!r}") + logger.info(f"Generating stubs for: {prefixes}") + logger.info(f"Writing stubs to: {output_dir}") + + jmodules = [import_module(prefix) for prefix in prefixes] + stubgenj.generateJavaStubs( + jmodules, + useStubsSuffix=False, + outputDir=str(output_dir), + jpypeJPackageStubs=False, + includeJavadoc=include_javadoc, + ) + + output_dir = Path(output_dir) + if add_runtime_imports: + logger.info("Adding runtime imports to generated stubs") + for stub in output_dir.rglob("*.pyi"): + stub_ast = ast.parse(stub.read_text()) + members = {node.name for node in stub_ast.body if hasattr(node, "name")} + if members == {"__module_protocol__"}: + # this is simply a module stub... no exports + if remove_namespace_only_stubs: + logger.info("Removing namespace only stub %s", stub) + stub.unlink() + continue + if add_runtime_imports: + real_import = stub.with_suffix(".py") + endpoint_args = ", ".join(repr(x) for x in endpoints) + real_import.write_text(INIT_TEMPLATE.format(endpoints=endpoint_args)) + + ruff_check(output_dir.absolute()) + + +# the "real" init file that goes into the stub package +INIT_TEMPLATE = """\ +# this file was autogenerated by scyjava-stubgen +# it creates a __getattr__ function that will dynamically import +# the requested class from the Java namespace corresponding to this module. +# see scyjava._stubs for implementation details. +from scyjava._stubs import dynamic_import + +__all__, __getattr__ = dynamic_import(__name__, __file__, {endpoints}) +""" + + +def ruff_check(output: Path, select: str = "E,W,F,I,UP,C4,B,RUF,TC,TID") -> None: + """Run ruff check and format on the generated stubs.""" + if not shutil.which("ruff"): + return + + py_files = [str(x) for x in chain(output.rglob("*.py"), output.rglob("*.pyi"))] + logger.info( + "Running ruff check on %d generated stubs in % s", + len(py_files), + str(output), + ) + subprocess.run( + [ + "ruff", + "check", + *py_files, + "--quiet", + "--fix-only", + "--unsafe-fixes", + f"--select={select}", + ] + ) + logger.info("Running ruff format") + subprocess.run(["ruff", "format", *py_files, "--quiet"]) + + +def list_top_level_packages(jar_path: str) -> set[str]: + """Inspect a JAR file and return the set of top-level Java package names.""" + packages: set[str] = set() + with ZipFile(jar_path, "r") as jar: + # find all classes + class_dirs = { + entry.parent + for x in jar.namelist() + if (entry := PurePath(x)).suffix == ".class" + } + + roots: set[PurePath] = set() + for p in sorted(class_dirs, key=lambda p: len(p.parts)): + # If none of the already accepted roots is a parent of p, keep p + if not any(root in p.parents for root in roots): + roots.add(p) + packages.update({str(p).replace(os.sep, ".") for p in roots}) + + return packages From a8b2da72273ddeffde466107ea8ba8daeba259ab Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:17:25 -0400 Subject: [PATCH 03/29] remove import --- src/scyjava/_stubs/_genstubs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index be800bd..74ac78c 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -12,7 +12,6 @@ from unittest.mock import patch from zipfile import ZipFile -import cjdk import scyjava import scyjava.config import stubgenj From 686739b56c0645e1dc57f129169098a283f63e58 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:46:32 -0400 Subject: [PATCH 04/29] add test --- src/scyjava/_stubs/_genstubs.py | 4 +++ tests/test_stubgen.py | 56 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/test_stubgen.py diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 74ac78c..b2c1910 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -69,6 +69,10 @@ def generate_stubs( stub generator wrote to it (and therefore inaccurate). """ import jpype + + # FIXME: either remove the _JImportLoader from sys.meta_path after this is done + # (if it wasn't there to begin with), or replace the import_module calls below + # with a more direct JPackage call import jpype.imports startJVM = jpype.startJVM diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py new file mode 100644 index 0000000..4b7a0bb --- /dev/null +++ b/tests/test_stubgen.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING +from unittest.mock import patch + +import jpype +import jpype.imports + +import scyjava +from scyjava._stubs import _cli + +if TYPE_CHECKING: + from pathlib import Path + + import pytest + + +def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "scyjava-stubgen", + "org.scijava:parsington:3.1.0", + "--output-dir", + str(tmp_path), + ], + ) + _cli.main() + + # remove the `jpype.imports` magic from the import system if present + mp = [x for x in sys.meta_path if not isinstance(x, jpype.imports._JImportLoader)] + monkeypatch.setattr(sys, "meta_path", mp) + + # add tmp_path to the import path + monkeypatch.setattr(sys, "path", [str(tmp_path)]) + + # first cleanup to make sure we are not importing from the cache + sys.modules.pop("org", None) + sys.modules.pop("org.scijava", None) + sys.modules.pop("org.scijava.parsington", None) + # make sure the stubgen command works and that we can now impmort stuff + + with patch.object(scyjava._jvm, "start_jvm") as mock_start_jvm: + from org.scijava.parsington import Function + + assert Function is not None + assert repr(Function) == "" + # ensure that no calls to start_jvm were made + mock_start_jvm.assert_not_called() + + # only after instantiating the class should we have a call to start_jvm + func = Function(1) + mock_start_jvm.assert_called_once() + assert isinstance(func, jpype.JObject) From 7fe31af6f6d0c72bd9ae1bfb8a77ee44a85033a8 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:47:27 -0400 Subject: [PATCH 05/29] add comment to clarify stubgen command execution in test --- tests/test_stubgen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py index 4b7a0bb..bb7fd9b 100644 --- a/tests/test_stubgen.py +++ b/tests/test_stubgen.py @@ -17,6 +17,7 @@ def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # run the stubgen command as if it was run from the command line monkeypatch.setattr( sys, "argv", From afcc7a7bb30da19cc965e3079f4a32be517471df Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:57:34 -0400 Subject: [PATCH 06/29] refactor: clean up jpype imports in stubgen test and main module --- src/scyjava/_stubs/_genstubs.py | 19 +++++++++++++------ tests/test_stubgen.py | 1 - 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index b2c1910..b7d5b46 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -8,10 +8,13 @@ from importlib import import_module from itertools import chain from pathlib import Path, PurePath +import sys from typing import TYPE_CHECKING, Any from unittest.mock import patch from zipfile import ZipFile +import jpype + import scyjava import scyjava.config import stubgenj @@ -70,11 +73,6 @@ def generate_stubs( """ import jpype - # FIXME: either remove the _JImportLoader from sys.meta_path after this is done - # (if it wasn't there to begin with), or replace the import_module calls below - # with a more direct JPackage call - import jpype.imports - startJVM = jpype.startJVM scyjava.config.endpoints.extend(endpoints) @@ -99,7 +97,16 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: logger.info(f"Generating stubs for: {prefixes}") logger.info(f"Writing stubs to: {output_dir}") - jmodules = [import_module(prefix) for prefix in prefixes] + metapath = sys.meta_path + try: + import jpype.imports + + jmodules = [import_module(prefix) for prefix in prefixes] + finally: + # remove the jpype.imports magic from the import system + # if it wasn't there to begin with + sys.meta_path = metapath + stubgenj.generateJavaStubs( jmodules, useStubsSuffix=False, diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py index bb7fd9b..300ee7f 100644 --- a/tests/test_stubgen.py +++ b/tests/test_stubgen.py @@ -5,7 +5,6 @@ from unittest.mock import patch import jpype -import jpype.imports import scyjava from scyjava._stubs import _cli From a5cacc809e0db42f31f44646d4646f176894fe79 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:58:39 -0400 Subject: [PATCH 07/29] remove unused jpype import from _genstubs.py --- src/scyjava/_stubs/_genstubs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index b7d5b46..22c0eca 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -13,8 +13,6 @@ from unittest.mock import patch from zipfile import ZipFile -import jpype - import scyjava import scyjava.config import stubgenj From ab1bc2d6579999deb572ee13288d86a3e24321af Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Tue, 22 Apr 2025 23:59:13 -0400 Subject: [PATCH 08/29] fix: add future annotations import to _cli.py --- src/scyjava/_stubs/_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scyjava/_stubs/_cli.py b/src/scyjava/_stubs/_cli.py index eeb6970..3a4d7df 100644 --- a/src/scyjava/_stubs/_cli.py +++ b/src/scyjava/_stubs/_cli.py @@ -1,5 +1,7 @@ """The scyjava-stubs executable.""" +from __future__ import annotations + import argparse import importlib import importlib.util From 11649faea090a8a65dc92f2eaa853e3c5f6249b7 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 23 Apr 2025 08:16:30 -0400 Subject: [PATCH 09/29] refactor: enhance dynamic_import function to accept base_prefix and improve stub generation --- src/scyjava/_stubs/_dynamic_import.py | 23 ++++++++++++----------- src/scyjava/_stubs/_genstubs.py | 17 ++++++++++++++--- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py index af50065..f62407b 100644 --- a/src/scyjava/_stubs/_dynamic_import.py +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -1,11 +1,14 @@ import ast from logging import warning from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Sequence def dynamic_import( - module_name: str, module_file: str, *endpoints: str + module_name: str, + module_file: str, + endpoints: Sequence[str] = (), + base_prefix: str = "", ) -> tuple[list[str], Callable[[str], Any]]: import scyjava import scyjava.config @@ -35,27 +38,25 @@ def module_getattr(name: str, mod_name: str = module_name) -> Any: if module_all and name not in module_all: raise AttributeError(f"module {module_name!r} has no attribute {name!r}") - # this strip is important... and tricky, because it depends on the - # namespace that we intend to install the stubs into. - install_path = "scyjava.types." - if mod_name.startswith(install_path): - mod_name = mod_name[len(install_path) :] + # cut the mod_name to only the part including the base_prefix and after + if base_prefix in mod_name: + mod_name = mod_name[mod_name.index(base_prefix) :] - full_name = f"{mod_name}.{name}" + class_path = f"{mod_name}.{name}" class ProxyMeta(type): def __repr__(self) -> str: - return f"" + return f"" class Proxy(metaclass=ProxyMeta): def __new__(_cls_, *args: Any, **kwargs: Any) -> Any: - cls = scyjava.jimport(full_name) + cls = scyjava.jimport(class_path) return cls(*args, **kwargs) Proxy.__name__ = name Proxy.__qualname__ = name Proxy.__module__ = module_name - Proxy.__doc__ = f"Proxy for {full_name}" + Proxy.__doc__ = f"Proxy for {class_path}" return Proxy return module_all, module_getattr diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 22c0eca..754b33f 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -116,6 +116,7 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: output_dir = Path(output_dir) if add_runtime_imports: logger.info("Adding runtime imports to generated stubs") + for stub in output_dir.rglob("*.pyi"): stub_ast = ast.parse(stub.read_text()) members = {node.name for node in stub_ast.body if hasattr(node, "name")} @@ -127,8 +128,13 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: continue if add_runtime_imports: real_import = stub.with_suffix(".py") - endpoint_args = ", ".join(repr(x) for x in endpoints) - real_import.write_text(INIT_TEMPLATE.format(endpoints=endpoint_args)) + base_prefix = stub.relative_to(output_dir).parts[0] + real_import.write_text( + INIT_TEMPLATE.format( + endpoints=repr(endpoints), + base_prefix=repr(base_prefix), + ) + ) ruff_check(output_dir.absolute()) @@ -141,7 +147,12 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: # see scyjava._stubs for implementation details. from scyjava._stubs import dynamic_import -__all__, __getattr__ = dynamic_import(__name__, __file__, {endpoints}) +__all__, __getattr__ = dynamic_import( + __name__, + __file__, + endpoints={endpoints}, + base_prefix={base_prefix}, +) """ From 2cb4836c370a678b307849891300d28a9777b6e1 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 23 Apr 2025 08:43:06 -0400 Subject: [PATCH 10/29] refactor: rename dynamic_import to setup_java_imports and update usage in stubs --- src/scyjava/_stubs/__init__.py | 4 +-- src/scyjava/_stubs/_dynamic_import.py | 41 ++++++++++++++++++++++++++- src/scyjava/_stubs/_genstubs.py | 13 +++++++-- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/scyjava/_stubs/__init__.py b/src/scyjava/_stubs/__init__.py index 4f42a0b..d6a5e7c 100644 --- a/src/scyjava/_stubs/__init__.py +++ b/src/scyjava/_stubs/__init__.py @@ -1,4 +1,4 @@ -from ._dynamic_import import dynamic_import +from ._dynamic_import import setup_java_imports from ._genstubs import generate_stubs -__all__ = ["dynamic_import", "generate_stubs"] +__all__ = ["setup_java_imports", "generate_stubs"] diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py index f62407b..3ce6669 100644 --- a/src/scyjava/_stubs/_dynamic_import.py +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -4,12 +4,51 @@ from typing import Any, Callable, Sequence -def dynamic_import( +def setup_java_imports( module_name: str, module_file: str, endpoints: Sequence[str] = (), base_prefix: str = "", ) -> tuple[list[str], Callable[[str], Any]]: + """Setup a module to dynamically import Java class names. + + This function creates a `__getattr__` function that, when called, will dynamically + import the requested class from the Java namespace corresponding to this module. + + :param module_name: The dotted name/identifier of the module that is calling this + function (usually `__name__` in the calling module). + :param module_file: The path to the module file (usually `__file__` in the calling + module). + :param endpoints: A list of Java endpoints to add to the scyjava configuration. + :param base_prefix: The base prefix for the Java package name. This is used when + determining the Java class path for the requested class. The java class path + will be truncated to only the part including the base_prefix and after. This + makes it possible to embed a module in a subpackage (like `scyjava.types`) and + still have the correct Java class path. + :return: A 2-tuple containing: + - A list of all classes in the module (as defined in the stub file), to be + assigned to `__all__`. + - A callable that takes a class name and returns a proxy for the Java class. + This callable should be assigned to `__getattr__` in the calling module. + The proxy object, when called, will start the JVM, import the Java class, + and return an instance of the class. The JVM will *only* be started when + the object is called. + + Example: + If the module calling this function is named `scyjava.types.org.scijava.parsington`, + then it should invoke this function as: + + .. code-block:: python + + from scyjava._stubs import setup_java_imports + + __all__, __getattr__ = setup_java_imports( + __name__, + __file__, + endpoints=["org.scijava:parsington:3.1.0"], + base_prefix="org" + ) + """ import scyjava import scyjava.config diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 754b33f..cb6bef2 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -15,7 +15,6 @@ import scyjava import scyjava.config -import stubgenj if TYPE_CHECKING: from collections.abc import Sequence @@ -69,6 +68,14 @@ def generate_stubs( the `__init__.pyi` for any given module will be whatever whatever the *last* stub generator wrote to it (and therefore inaccurate). """ + try: + import stubgenj + except ImportError as e: + raise ImportError( + "stubgenj is not installed, but is required to generate java stubs. " + "Please install it with `pip/conda install stubgenj`." + ) from e + import jpype startJVM = jpype.startJVM @@ -145,9 +152,9 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: # it creates a __getattr__ function that will dynamically import # the requested class from the Java namespace corresponding to this module. # see scyjava._stubs for implementation details. -from scyjava._stubs import dynamic_import +from scyjava._stubs import setup_java_imports -__all__, __getattr__ = dynamic_import( +__all__, __getattr__ = setup_java_imports( __name__, __file__, endpoints={endpoints}, From 71f761ed3972b484ae064e73354336696412b101 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 23 Apr 2025 08:43:58 -0400 Subject: [PATCH 11/29] reword --- src/scyjava/_stubs/_dynamic_import.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py index 3ce6669..cc62641 100644 --- a/src/scyjava/_stubs/_dynamic_import.py +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -13,7 +13,8 @@ def setup_java_imports( """Setup a module to dynamically import Java class names. This function creates a `__getattr__` function that, when called, will dynamically - import the requested class from the Java namespace corresponding to this module. + import the requested class from the Java namespace corresponding to the calling + module. :param module_name: The dotted name/identifier of the module that is calling this function (usually `__name__` in the calling module). From 65cc471a5df976c61f275a5bf42a7abd066b168b Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 24 Apr 2025 21:26:03 -0400 Subject: [PATCH 12/29] feat: add Hatchling build hook for generating Java stubs --- pyproject.toml | 13 +++++------ src/scyjava/_stubs/_hatchling.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 src/scyjava/_stubs/_hatchling.py diff --git a/pyproject.toml b/pyproject.toml index b36f2bf..0d7fb64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,12 +32,7 @@ classifiers = [ # NB: Keep this in sync with environment.yml AND dev-environment.yml! requires-python = ">=3.9" -dependencies = [ - "jpype1 >= 1.3.0", - "jgo", - "cjdk", - "stubgenj", -] +dependencies = ["jpype1 >= 1.3.0", "jgo", "cjdk", "stubgenj"] [project.optional-dependencies] # NB: Keep this in sync with dev-environment.yml! @@ -51,12 +46,16 @@ dev = [ "pandas", "ruff", "toml", - "validate-pyproject[all]" + "validate-pyproject[all]", ] [project.scripts] scyjava-stubgen = "scyjava._stubs._cli:main" +[project.entry-points.hatch] +mypyc = "scyjava._stubs._hatchling" + + [project.urls] homepage = "https://github.com/scijava/scyjava" documentation = "https://github.com/scijava/scyjava/blob/main/README.md" diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling.py new file mode 100644 index 0000000..d4c1ade --- /dev/null +++ b/src/scyjava/_stubs/_hatchling.py @@ -0,0 +1,38 @@ +"""Hatchling build hook for generating Java stubs.""" + +import shutil +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.plugin import hookimpl + + +from scyjava._stubs._genstubs import generate_stubs + + +class ScyjavaBuildHook(BuildHookInterface): + """Custom build hook for generating Java stubs.""" + + PLUGIN_NAME = "scyjava" + + def initialize(self, version: str, build_data: dict) -> None: + """Initialize the build hook with the version and build data.""" + breakpoint() + if self.target_name != "wheel": + return + dest = Path(self.root, "src") + shutil.rmtree(dest, ignore_errors=True) # remove the old stubs + + # actually build the stubs + coord = f"{self.config['maven_coord']}:{self.metadata.version}" + prefixes = self.config.get("prefixes", []) + generate_stubs(endpoints=[coord], prefixes=prefixes, output_dir=dest) + + # add all packages to the build config + packages = [str(x.relative_to(self.root)) for x in dest.iterdir()] + self.build_config.target_config.setdefault("packages", packages) + + +@hookimpl +def hatch_register_build_hook(): + return ScyjavaBuildHook From 6e4181e77c360920545651ebb1b2b652cc02db52 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 24 Apr 2025 21:54:57 -0400 Subject: [PATCH 13/29] wip --- src/scyjava/_jvm.py | 2 +- src/scyjava/_stubs/_hatchling.py | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/scyjava/_jvm.py b/src/scyjava/_jvm.py index 2035e34..d8a5bed 100644 --- a/src/scyjava/_jvm.py +++ b/src/scyjava/_jvm.py @@ -373,7 +373,7 @@ def is_awt_initialized() -> bool: return False Thread = scyjava.jimport("java.lang.Thread") threads = Thread.getAllStackTraces().keySet() - return any(t.getName().startsWith("AWT-") for t in threads) + return any(str(t.getName()).startswith("AWT-") for t in threads) def when_jvm_starts(f) -> None: diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling.py index d4c1ade..140b901 100644 --- a/src/scyjava/_stubs/_hatchling.py +++ b/src/scyjava/_stubs/_hatchling.py @@ -1,5 +1,6 @@ """Hatchling build hook for generating Java stubs.""" +import logging import shutil from pathlib import Path @@ -9,6 +10,8 @@ from scyjava._stubs._genstubs import generate_stubs +logger = logging.getLogger("scyjava") + class ScyjavaBuildHook(BuildHookInterface): """Custom build hook for generating Java stubs.""" @@ -17,20 +20,22 @@ class ScyjavaBuildHook(BuildHookInterface): def initialize(self, version: str, build_data: dict) -> None: """Initialize the build hook with the version and build data.""" - breakpoint() if self.target_name != "wheel": return - dest = Path(self.root, "src") - shutil.rmtree(dest, ignore_errors=True) # remove the old stubs - # actually build the stubs - coord = f"{self.config['maven_coord']}:{self.metadata.version}" + endpoints = self.config.get("maven_coordinates", []) + if not endpoints: + logger.warning("No maven coordinates provided. Skipping stub generation.") + return + prefixes = self.config.get("prefixes", []) - generate_stubs(endpoints=[coord], prefixes=prefixes, output_dir=dest) + dest = Path(self.root, "src", "scyjava", "types") - # add all packages to the build config - packages = [str(x.relative_to(self.root)) for x in dest.iterdir()] - self.build_config.target_config.setdefault("packages", packages) + # actually build the stubs + generate_stubs(endpoints=endpoints, prefixes=prefixes, output_dir=dest) + print(f"Generated stubs for {endpoints} in {dest}") + # add all new packages to the build config + build_data["artifacts"].append("src/scyjava/types") @hookimpl From 6e92b13720484ee55f33b92c8c9929ff5c52dbef Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 25 Apr 2025 08:05:33 -0400 Subject: [PATCH 14/29] fix inclusion --- src/scyjava/_stubs/_hatchling.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling.py index 140b901..52b18e2 100644 --- a/src/scyjava/_stubs/_hatchling.py +++ b/src/scyjava/_stubs/_hatchling.py @@ -30,12 +30,18 @@ def initialize(self, version: str, build_data: dict) -> None: prefixes = self.config.get("prefixes", []) dest = Path(self.root, "src", "scyjava", "types") + (dest.parent / "py.typed").touch() # actually build the stubs - generate_stubs(endpoints=endpoints, prefixes=prefixes, output_dir=dest) + generate_stubs( + endpoints=endpoints, + prefixes=prefixes, + output_dir=dest, + remove_namespace_only_stubs=True, + ) print(f"Generated stubs for {endpoints} in {dest}") # add all new packages to the build config - build_data["artifacts"].append("src/scyjava/types") + build_data["force_include"].update({str(dest.parent): "scyjava"}) @hookimpl From 0d231cccf93dc634bda853647252040063f66cdd Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 25 Apr 2025 08:08:06 -0400 Subject: [PATCH 15/29] add docs --- src/scyjava/_stubs/_hatchling.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling.py index 52b18e2..866f829 100644 --- a/src/scyjava/_stubs/_hatchling.py +++ b/src/scyjava/_stubs/_hatchling.py @@ -1,7 +1,23 @@ -"""Hatchling build hook for generating Java stubs.""" +"""Hatchling build hook for generating Java stubs. + +To use this hook, add the following to your `pyproject.toml`: + +```toml +[build-system] +requires = ["hatchling", "scyjava"] +build-backend = "hatchling.build" + +[tool.hatch.build.hooks.scyjava] +maven_coordinates = ["org.scijava:parsington:3.1.0"] +prefixes = ["org.scijava"] # optional ... can be auto-determined from the jar files +``` + +This will generate stubs for the given maven coordinates and prefixes. The generated +stubs will be placed in `src/scyjava/types` and will be included in the wheel package. +This hook is only run when building a wheel package. +""" import logging -import shutil from pathlib import Path from hatchling.builders.hooks.plugin.interface import BuildHookInterface From 79524b0065c0e6bdbb57c63d9d2cbe9bc8cc6c8b Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 25 Apr 2025 08:17:38 -0400 Subject: [PATCH 16/29] setuptools plugin stub --- pyproject.toml | 5 +- .../{_hatchling.py => _hatchling_plugin.py} | 0 src/scyjava/_stubs/_setuptools_plugin.py | 79 +++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) rename src/scyjava/_stubs/{_hatchling.py => _hatchling_plugin.py} (100%) create mode 100644 src/scyjava/_stubs/_setuptools_plugin.py diff --git a/pyproject.toml b/pyproject.toml index 0d7fb64..47f3ffb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,8 +53,9 @@ dev = [ scyjava-stubgen = "scyjava._stubs._cli:main" [project.entry-points.hatch] -mypyc = "scyjava._stubs._hatchling" - +scyjava = "scyjava._stubs._hatchling_plugin" +[project.entry-points."distutils.commands"] +build_py = "scyjava_stubgen.build:build_py" [project.urls] homepage = "https://github.com/scijava/scyjava" diff --git a/src/scyjava/_stubs/_hatchling.py b/src/scyjava/_stubs/_hatchling_plugin.py similarity index 100% rename from src/scyjava/_stubs/_hatchling.py rename to src/scyjava/_stubs/_hatchling_plugin.py diff --git a/src/scyjava/_stubs/_setuptools_plugin.py b/src/scyjava/_stubs/_setuptools_plugin.py new file mode 100644 index 0000000..e9386f9 --- /dev/null +++ b/src/scyjava/_stubs/_setuptools_plugin.py @@ -0,0 +1,79 @@ +"""Setuptools build hook for generating Java stubs. + +To use this hook, add the following to your `pyproject.toml`: + +```toml +[build-system] +requires = ["setuptools>=69", "wheel", "scyjava"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.cmdclass] +build_py = "scyjava_stubgen.build:build_py" +# optional project-specific defaults +maven_coordinates = ["org.scijava:parsington:3.1.0"] +prefixes = ["org.scijava"] +``` + +This will generate stubs for the given maven coordinates and prefixes. The generated +stubs will be placed in `src/scyjava/types` and will be included in the wheel package. +This hook is only run when building a wheel package. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import List + +from setuptools.command.build_py import build_py as _build_py +from scyjava._stubs._genstubs import generate_stubs + +log = logging.getLogger("scyjava") + + +class build_py(_build_py): # type: ignore[misc] + """ + A drop-in replacement for setuptools' build_py that + generates Java type stubs before Python sources are copied + into *build_lib*. + """ + + # expose two optional CLI/pyproject options so users can override defaults + user_options: List[tuple[str, str | None, str]] = _build_py.user_options + [ + ("maven-coordinates=", None, "List of Maven coordinates to stub"), + ("prefixes=", None, "Java package prefixes to include"), + ] + + def initialize_options(self) -> None: # noqa: D401 + super().initialize_options() + self.maven_coordinates: list[str] | None = None + self.prefixes: list[str] | None = None + + def finalize_options(self) -> None: # noqa: D401 + """Fill in options that may come from pyproject metadata.""" + super().finalize_options() + dist = self.distribution # alias + if self.maven_coordinates is None: + self.maven_coordinates = getattr(dist, "maven_coordinates", []) + if self.prefixes is None: + self.prefixes = getattr(dist, "prefixes", []) + + def run(self) -> None: # noqa: D401 + """Generate stubs, then let the normal build_py proceed.""" + if self.maven_coordinates: + dest = Path(self.build_lib, "scyjava", "types") + dest.parent.mkdir(parents=True, exist_ok=True) + (dest.parent / "py.typed").touch() + + generate_stubs( + endpoints=self.maven_coordinates, + prefixes=self.prefixes, + output_dir=dest, + remove_namespace_only_stubs=True, + ) + log.info("Generated stubs for %s", ", ".join(self.maven_coordinates)) + + # make sure the wheel knows about them + self.package_data.setdefault("scyjava", []).append("types/**/*.pyi") + + super().run() From 51937b587562d13922b4dfd944fd142c5c5fee6a Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 30 Apr 2025 20:17:11 -0400 Subject: [PATCH 17/29] remove repr test --- tests/test_stubgen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py index 300ee7f..0d4c28d 100644 --- a/tests/test_stubgen.py +++ b/tests/test_stubgen.py @@ -46,7 +46,6 @@ def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: from org.scijava.parsington import Function assert Function is not None - assert repr(Function) == "" # ensure that no calls to start_jvm were made mock_start_jvm.assert_not_called() From 192be35b1c7ad073064589a348af4be96b65723e Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 1 May 2025 09:05:49 -0400 Subject: [PATCH 18/29] skip in jep --- tests/test_stubgen.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py index 0d4c28d..ad4a74a 100644 --- a/tests/test_stubgen.py +++ b/tests/test_stubgen.py @@ -5,6 +5,7 @@ from unittest.mock import patch import jpype +import pytest import scyjava from scyjava._stubs import _cli @@ -12,9 +13,11 @@ if TYPE_CHECKING: from pathlib import Path - import pytest - +@pytest.mark.skipif( + scyjava.config.mode != scyjava.config.Mode.JPYPE, + reason="Stubgen not supported in JEP", +) def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: # run the stubgen command as if it was run from the command line monkeypatch.setattr( From e714abb7a00df87550fd3fee13faaeb55593ce64 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 1 May 2025 09:12:41 -0400 Subject: [PATCH 19/29] remove setuptools plugin --- src/scyjava/_stubs/_setuptools_plugin.py | 79 ------------------------ 1 file changed, 79 deletions(-) delete mode 100644 src/scyjava/_stubs/_setuptools_plugin.py diff --git a/src/scyjava/_stubs/_setuptools_plugin.py b/src/scyjava/_stubs/_setuptools_plugin.py deleted file mode 100644 index e9386f9..0000000 --- a/src/scyjava/_stubs/_setuptools_plugin.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Setuptools build hook for generating Java stubs. - -To use this hook, add the following to your `pyproject.toml`: - -```toml -[build-system] -requires = ["setuptools>=69", "wheel", "scyjava"] -build-backend = "setuptools.build_meta" - -[tool.setuptools.cmdclass] -build_py = "scyjava_stubgen.build:build_py" -# optional project-specific defaults -maven_coordinates = ["org.scijava:parsington:3.1.0"] -prefixes = ["org.scijava"] -``` - -This will generate stubs for the given maven coordinates and prefixes. The generated -stubs will be placed in `src/scyjava/types` and will be included in the wheel package. -This hook is only run when building a wheel package. -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import List - -from setuptools.command.build_py import build_py as _build_py -from scyjava._stubs._genstubs import generate_stubs - -log = logging.getLogger("scyjava") - - -class build_py(_build_py): # type: ignore[misc] - """ - A drop-in replacement for setuptools' build_py that - generates Java type stubs before Python sources are copied - into *build_lib*. - """ - - # expose two optional CLI/pyproject options so users can override defaults - user_options: List[tuple[str, str | None, str]] = _build_py.user_options + [ - ("maven-coordinates=", None, "List of Maven coordinates to stub"), - ("prefixes=", None, "Java package prefixes to include"), - ] - - def initialize_options(self) -> None: # noqa: D401 - super().initialize_options() - self.maven_coordinates: list[str] | None = None - self.prefixes: list[str] | None = None - - def finalize_options(self) -> None: # noqa: D401 - """Fill in options that may come from pyproject metadata.""" - super().finalize_options() - dist = self.distribution # alias - if self.maven_coordinates is None: - self.maven_coordinates = getattr(dist, "maven_coordinates", []) - if self.prefixes is None: - self.prefixes = getattr(dist, "prefixes", []) - - def run(self) -> None: # noqa: D401 - """Generate stubs, then let the normal build_py proceed.""" - if self.maven_coordinates: - dest = Path(self.build_lib, "scyjava", "types") - dest.parent.mkdir(parents=True, exist_ok=True) - (dest.parent / "py.typed").touch() - - generate_stubs( - endpoints=self.maven_coordinates, - prefixes=self.prefixes, - output_dir=dest, - remove_namespace_only_stubs=True, - ) - log.info("Generated stubs for %s", ", ".join(self.maven_coordinates)) - - # make sure the wheel knows about them - self.package_data.setdefault("scyjava", []).append("types/**/*.pyi") - - super().run() From a18d9d908e2e11263679eb09ae3b3b14d17553ce Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 22 Aug 2025 19:17:00 -0400 Subject: [PATCH 20/29] remove hatch plugin --- pyproject.toml | 5 -- src/scyjava/_stubs/_genstubs.py | 2 +- src/scyjava/_stubs/_hatchling_plugin.py | 65 ------------------------- 3 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 src/scyjava/_stubs/_hatchling_plugin.py diff --git a/pyproject.toml b/pyproject.toml index 3471c09..76c8882 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,11 +49,6 @@ dev = [ [project.scripts] scyjava-stubgen = "scyjava._stubs._cli:main" -[project.entry-points.hatch] -scyjava = "scyjava._stubs._hatchling_plugin" -[project.entry-points."distutils.commands"] -build_py = "scyjava_stubgen.build:build_py" - [project.urls] homepage = "https://github.com/scijava/scyjava" documentation = "https://github.com/scijava/scyjava/blob/main/README.md" diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index cb6bef2..1576d64 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -5,10 +5,10 @@ import os import shutil import subprocess +import sys from importlib import import_module from itertools import chain from pathlib import Path, PurePath -import sys from typing import TYPE_CHECKING, Any from unittest.mock import patch from zipfile import ZipFile diff --git a/src/scyjava/_stubs/_hatchling_plugin.py b/src/scyjava/_stubs/_hatchling_plugin.py deleted file mode 100644 index 866f829..0000000 --- a/src/scyjava/_stubs/_hatchling_plugin.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Hatchling build hook for generating Java stubs. - -To use this hook, add the following to your `pyproject.toml`: - -```toml -[build-system] -requires = ["hatchling", "scyjava"] -build-backend = "hatchling.build" - -[tool.hatch.build.hooks.scyjava] -maven_coordinates = ["org.scijava:parsington:3.1.0"] -prefixes = ["org.scijava"] # optional ... can be auto-determined from the jar files -``` - -This will generate stubs for the given maven coordinates and prefixes. The generated -stubs will be placed in `src/scyjava/types` and will be included in the wheel package. -This hook is only run when building a wheel package. -""" - -import logging -from pathlib import Path - -from hatchling.builders.hooks.plugin.interface import BuildHookInterface -from hatchling.plugin import hookimpl - - -from scyjava._stubs._genstubs import generate_stubs - -logger = logging.getLogger("scyjava") - - -class ScyjavaBuildHook(BuildHookInterface): - """Custom build hook for generating Java stubs.""" - - PLUGIN_NAME = "scyjava" - - def initialize(self, version: str, build_data: dict) -> None: - """Initialize the build hook with the version and build data.""" - if self.target_name != "wheel": - return - - endpoints = self.config.get("maven_coordinates", []) - if not endpoints: - logger.warning("No maven coordinates provided. Skipping stub generation.") - return - - prefixes = self.config.get("prefixes", []) - dest = Path(self.root, "src", "scyjava", "types") - (dest.parent / "py.typed").touch() - - # actually build the stubs - generate_stubs( - endpoints=endpoints, - prefixes=prefixes, - output_dir=dest, - remove_namespace_only_stubs=True, - ) - print(f"Generated stubs for {endpoints} in {dest}") - # add all new packages to the build config - build_data["force_include"].update({str(dest.parent): "scyjava"}) - - -@hookimpl -def hatch_register_build_hook(): - return ScyjavaBuildHook From b53e795b5e1dac8e3c2037ce91d41e06c59e3233 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 22 Aug 2025 19:18:02 -0400 Subject: [PATCH 21/29] newlines --- pyproject.toml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 76c8882..ccb3fb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "scyjava" version = "1.12.2.dev0" description = "Supercharged Java access from Python" license = "Unlicense" -authors = [{name = "SciJava developers", email = "ctrueden@wisc.edu"}] +authors = [{ name = "SciJava developers", email = "ctrueden@wisc.edu" }] readme = "README.md" keywords = ["java", "maven", "cross-language"] classifiers = [ @@ -31,7 +31,12 @@ classifiers = [ ] requires-python = ">=3.9" -dependencies = ["jpype1 >= 1.3.0", "jgo", "cjdk", "stubgenj"] +dependencies = [ + "jpype1 >= 1.3.0", + "jgo", + "cjdk", + "stubgenj", +] [dependency-groups] dev = [ @@ -57,7 +62,7 @@ download = "https://pypi.org/project/scyjava/" tracker = "https://github.com/scijava/scyjava/issues" [tool.setuptools] -package-dir = {"" = "src"} +package-dir = { "" = "src" } include-package-data = false [tool.setuptools.packages.find] From ed0add6b06a71dd729fe502c6c8ec16588e9e5e1 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 22 Aug 2025 19:35:07 -0400 Subject: [PATCH 22/29] update docs --- src/scyjava/_stubs/_cli.py | 10 ++++++++- src/scyjava/_stubs/_dynamic_import.py | 29 +++++++++++++++++++++++++++ src/scyjava/_stubs/_genstubs.py | 27 +++++++++++++++++++------ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/scyjava/_stubs/_cli.py b/src/scyjava/_stubs/_cli.py index 3a4d7df..7936529 100644 --- a/src/scyjava/_stubs/_cli.py +++ b/src/scyjava/_stubs/_cli.py @@ -1,4 +1,12 @@ -"""The scyjava-stubs executable.""" +"""The scyjava-stubs executable. + +Provides cli access to the `scyjava._stubs.generate_stubs` function. + +The only interesting additional things going on here is the choice of *where* the stubs +go by default. When using the CLI, they land in `scyjava.types` by default; see the +`_get_ouput_dir` helper function for details on how the output directory is resolved +from the CLI arguments. +""" from __future__ import annotations diff --git a/src/scyjava/_stubs/_dynamic_import.py b/src/scyjava/_stubs/_dynamic_import.py index cc62641..16e27d4 100644 --- a/src/scyjava/_stubs/_dynamic_import.py +++ b/src/scyjava/_stubs/_dynamic_import.py @@ -1,3 +1,24 @@ +"""Logic for using generated type stubs as runtime importable, with lazy JVM startup. + +Most often, the functionality here will be used as follows: + +``` +from scyjava._stubs import setup_java_imports + +__all__, __getattr__ = setup_java_imports( + __name__, + __file__, + endpoints=["org.scijava:parsington:3.1.0"], + base_prefix="org" +) +``` + +...and that little snippet is written into the generated stubs modules by the +`scyjava._stubs.generate_stubs` function. + +See docstring of `setup_java_imports` for details on how it works. +""" + import ast from logging import warning from pathlib import Path @@ -21,11 +42,14 @@ def setup_java_imports( :param module_file: The path to the module file (usually `__file__` in the calling module). :param endpoints: A list of Java endpoints to add to the scyjava configuration. + (Note that `scyjava._stubs.generate_stubs` will automatically add the necessary + endpoints for the generated stubs.) :param base_prefix: The base prefix for the Java package name. This is used when determining the Java class path for the requested class. The java class path will be truncated to only the part including the base_prefix and after. This makes it possible to embed a module in a subpackage (like `scyjava.types`) and still have the correct Java class path. + :return: A 2-tuple containing: - A list of all classes in the module (as defined in the stub file), to be assigned to `__all__`. @@ -57,6 +81,7 @@ def setup_java_imports( if ep not in scyjava.config.endpoints: scyjava.config.endpoints.append(ep) + # list intended to be assigned to `__all__` in the generated module. module_all = [] try: my_stub = Path(module_file).with_suffix(".pyi") @@ -75,6 +100,7 @@ def setup_java_imports( ) def module_getattr(name: str, mod_name: str = module_name) -> Any: + """Function intended to be assigned to __getattr__ in the generate module.""" if module_all and name not in module_all: raise AttributeError(f"module {module_name!r} has no attribute {name!r}") @@ -84,6 +110,9 @@ def module_getattr(name: str, mod_name: str = module_name) -> Any: class_path = f"{mod_name}.{name}" + # Generate a proxy type (with a nice repr) that + # delays the call to `jimport` until the last moment when type.__new__ is called + class ProxyMeta(type): def __repr__(self) -> str: return f"" diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 1576d64..3df24cb 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -1,3 +1,14 @@ +"""Type stub generation utilities using stubgen. + +This module provides utilities for generating type stubs for Java classes +using the stubgenj library. `stubgenj` must be installed for this to work +(it, in turn, only depends on JPype). + +See `generate_stubs` for most functionality. For the command-line tool, +see `scyjava._stubs.cli`, which provides a CLI interface for the `generate_stubs` +function. +""" + from __future__ import annotations import ast @@ -42,9 +53,11 @@ def generate_stubs( The prefixes to generate stubs for. This should be a list of Java class prefixes that you expect to find in the endpoints. For example, ["org.apache.commons"]. If not provided, the prefixes will be - automatically determined from the jar files provided by endpoints. + automatically determined from the jar files provided by endpoints (see the + `_list_top_level_packages` helper function). output_dir : str | Path, optional - The directory to write the generated stubs to. Defaults to "stubs". + The directory to write the generated stubs to. Defaults to "stubs" in the + current working directory. convert_strings : bool, optional Whether to cast Java strings to Python strings in the stubs. Defaults to True. NOTE: This leads to type stubs that may not be strictly accurate at runtime. @@ -58,8 +71,10 @@ def generate_stubs( Whether to include Javadoc in the generated stubs. Defaults to True. add_runtime_imports : bool, optional Whether to add runtime imports to the generated stubs. Defaults to True. - This is useful if you want to use the stubs as a runtime package with type - safety. + This is useful if you want to actually import the stubs as a runtime package + with type safety. The runtime import "magic" depends on the + `scyjava._stubs.setup_java_imports` function. See its documentation for + more details. remove_namespace_only_stubs : bool, optional Whether to remove stubs that export no names beyond a single `__module_protocol__`. This leaves some folders as PEP420 implicit namespace @@ -95,7 +110,7 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: ep_artifacts = tuple(ep.split(":")[1] for ep in endpoints) for j in cp.split(os.pathsep): if Path(j).name.startswith(ep_artifacts): - _prefixes.update(list_top_level_packages(j)) + _prefixes.update(_list_top_level_packages(j)) prefixes = sorted(_prefixes) logger.info(f"Using endpoints: {scyjava.config.endpoints!r}") @@ -189,7 +204,7 @@ def ruff_check(output: Path, select: str = "E,W,F,I,UP,C4,B,RUF,TC,TID") -> None subprocess.run(["ruff", "format", *py_files, "--quiet"]) -def list_top_level_packages(jar_path: str) -> set[str]: +def _list_top_level_packages(jar_path: str) -> set[str]: """Inspect a JAR file and return the set of top-level Java package names.""" packages: set[str] = set() with ZipFile(jar_path, "r") as jar: From 43cb06d2fadfc30cd60d5376fcbc9e1765ad1e4f Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 22 Aug 2025 19:54:45 -0400 Subject: [PATCH 23/29] initial --- METAFINDER_README.md | 81 ++++++++ src/scyjava/types/.gitignore | 3 +- src/scyjava/types/__init__.py | 358 ++++++++++++++++++++++++++++++++++ test_meta_finder.py | 113 +++++++++++ x.py | 4 + 5 files changed, 558 insertions(+), 1 deletion(-) create mode 100644 METAFINDER_README.md create mode 100644 src/scyjava/types/__init__.py create mode 100644 test_meta_finder.py create mode 100644 x.py diff --git a/METAFINDER_README.md b/METAFINDER_README.md new file mode 100644 index 0000000..a147b0d --- /dev/null +++ b/METAFINDER_README.md @@ -0,0 +1,81 @@ +# scyjava.types Meta Path Finder Infrastructure + +This module provides a complete meta path finder infrastructure for `scyjava.types` that: + +## Core Features + +1. **Intercepts imports** from `scyjava.types.XXX` using a custom meta path finder +2. **Calls a user-defined function** at import time to generate the module +3. **Executes the import** after the module is generated +4. **Uses `__name__` for portability** - works regardless of where the module is located + +## Key Components + +### ScyJavaTypesMetaFinder +- Implements the `find_spec()` method required by Python's import system +- Only handles imports that start with `scyjava.types.` +- Creates a `ModuleSpec` with our custom loader + +### ScyJavaTypesLoader +- Implements `create_module()` and `exec_module()` for the new import system +- Also implements `load_module()` for backward compatibility +- Calls the registered generator function to create modules + +### Module Generator Function +- User-provided function that receives the full module name +- Must return a `types.ModuleType` instance +- Called exactly once per unique import (modules are cached) + +## API Functions + +- `set_module_generator(func)` - Register your module generation function +- `get_registered_generator()` - Get the currently registered function +- `is_meta_finder_installed()` - Check if the meta finder is active +- `clear_generated_modules()` - Remove generated modules from cache +- `list_generated_modules()` - List all currently loaded generated modules + +## Usage Pattern + +```python +import scyjava.types + +def my_generator(module_name: str) -> types.ModuleType: + # module_name will be something like "scyjava.types.ArrayList" + module = types.ModuleType(module_name) + + # Extract class name from the module path + class_name = module_name.split('.')[-1] # "ArrayList" + + # Generate your class/content here + generated_class = create_java_stub(class_name) + setattr(module, class_name, generated_class) + + module.__file__ = f"" + return module + +# Register the generator +scyjava.types.set_module_generator(my_generator) + +# Now imports will trigger the generator +import scyjava.types.ArrayList as al_module +ArrayList = al_module.ArrayList + +# Or using from-import +from scyjava.types import HashMap as hm_module +HashMap = hm_module.HashMap +``` + +## Error Handling + +- If no generator is registered, imports raise `ImportError` +- If the generator returns `None`, imports raise `ImportError` +- Generator exceptions propagate to the import statement + +## Caching + +- Modules are automatically cached in `sys.modules` +- Subsequent imports of the same module return the cached version +- Use `clear_generated_modules()` to force regeneration + +This infrastructure provides the foundation for dynamic Java class stub generation +at import time, enabling type-safe imports with lazy initialization. diff --git a/src/scyjava/types/.gitignore b/src/scyjava/types/.gitignore index 5e7d273..3fa0a2d 100644 --- a/src/scyjava/types/.gitignore +++ b/src/scyjava/types/.gitignore @@ -1,4 +1,5 @@ # Ignore everything in this directory * -# Except this file +# Except this file and the __init__.py file !.gitignore +!__init__.py diff --git a/src/scyjava/types/__init__.py b/src/scyjava/types/__init__.py new file mode 100644 index 0000000..ac25d2c --- /dev/null +++ b/src/scyjava/types/__init__.py @@ -0,0 +1,358 @@ +""" +Dynamic type-safe imports for scyjava types with lazy initialization. + +This module provides a meta path finder that intercepts imports from scyjava.types +and dynamically generates the requested modules at import time. + +Usage: + 1. Set up a module generator function: + + def my_generator(module_name: str) -> types.ModuleType: + # Create and return a module based on module_name + module = types.ModuleType(module_name) + # ... populate module with classes/functions ... + return module + + 2. Register the generator: + + scyjava.types.set_module_generator(my_generator) + + 3. Import dynamically generated modules: + + import scyjava.types.SomeClass as some_class_module + SomeClass = some_class_module.SomeClass + + # Or use from import: + from scyjava.types import SomeClass as some_class_module + SomeClass = some_class_module.SomeClass + +The generator function will be called at import time with the full module name +(e.g., "scyjava.types.SomeClass") and should return a populated module. + +API Functions: + - set_module_generator(func): Register a module generator function + - get_registered_generator(): Get the currently registered generator + - is_meta_finder_installed(): Check if the meta finder is active + - clear_generated_modules(): Remove all generated modules from cache + - list_generated_modules(): List all currently loaded generated modules + +Example: + >>> import scyjava.types + >>> + >>> def java_stub_generator(module_name): + ... import types + ... module = types.ModuleType(module_name) + ... class_name = module_name.split('.')[-1] + ... # Generate Java class stub here... + ... java_class = create_java_stub(class_name) + ... setattr(module, class_name, java_class) + ... return module + ... + >>> scyjava.types.set_module_generator(java_stub_generator) + >>> + >>> # This will call java_stub_generator("scyjava.types.ArrayList") + >>> import scyjava.types.ArrayList as al_module + >>> ArrayList = al_module.ArrayList +""" + +import importlib.util +import sys +import types +from typing import Any, Callable, Optional, Sequence + +# The function that will be called to generate modules at import time +_module_generator: Optional[Callable[[str], Optional[types.ModuleType]]] = None + + +def set_module_generator(func: Callable[[str], Optional[types.ModuleType]]) -> None: + """ + Set the function that will be called to generate modules at import time. + + Args: + func: A callable that takes a module name (str) and returns a module + or None if the module cannot be generated. + """ + global _module_generator + _module_generator = func + + +class ScyJavaTypesMetaFinder: + """ + A meta path finder that intercepts imports from scyjava.types and + dynamically generates modules at import time. + """ + + @classmethod + def _get_base_package_name(cls) -> str: + """Get the base package name for portability.""" + # Use __name__ for portability - this will be 'scyjava.types' + return __name__ + + def find_spec( + self, + fullname: str, + path: Optional[Sequence[str]] = None, + target: Optional[types.ModuleType] = None, + ): + """ + Find a module spec for dynamic module generation. + + Args: + fullname: The fully qualified name of the module being imported + path: The search path (unused) + target: The target module (unused) + + Returns: + A ModuleSpec if this finder should handle the import, None otherwise + """ + base_package = self._get_base_package_name() + + # Only handle imports that start with our base package + if not fullname.startswith(base_package + "."): + return None + + # Don't handle the base package itself + if fullname == base_package: + return None + + # Create a module spec with our custom loader + spec = importlib.util.spec_from_loader( + fullname, ScyJavaTypesLoader(fullname), origin="dynamic" + ) + return spec + + +class ScyJavaTypesLoader: + """ + A module loader that calls the registered module generator function + to create modules dynamically at import time. + """ + + def __init__(self, fullname: str): + self.fullname = fullname + + def create_module(self, spec) -> Optional[types.ModuleType]: + """ + Create a module by calling the registered generator function. + + Args: + spec: The module spec + + Returns: + The generated module or None to use default module creation + """ + if _module_generator is None: + raise ImportError( + f"No module generator registered for {self.fullname}. " + f"Call scyjava.types.set_module_generator() first." + ) + + # Call the registered function to generate the module + module = _module_generator(self.fullname) + if module is None: + raise ImportError( + f"Module generator failed to create module: {self.fullname}" + ) + + return module + + def exec_module(self, module: types.ModuleType) -> None: + """ + Execute the module. Since our modules are generated dynamically, + there's nothing to execute here. + + Args: + module: The module to execute + """ + # Module is already fully initialized by the generator function + pass + + def load_module(self, fullname: str) -> types.ModuleType: + """ + Load a module (deprecated method, kept for compatibility). + + Args: + fullname: The fully qualified module name + + Returns: + The loaded module + """ + # Check if module is already in sys.modules + if fullname in sys.modules: + return sys.modules[fullname] + + # Create module spec and load it + spec = importlib.util.spec_from_loader(fullname, self, origin="dynamic") + if spec is None: + raise ImportError(f"Failed to create spec for module: {fullname}") + + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Failed to create module: {fullname}") + + # Add to sys.modules before exec + sys.modules[fullname] = module + + try: + if spec.loader: + spec.loader.exec_module(module) + except Exception: + # Remove from sys.modules if exec fails + sys.modules.pop(fullname, None) + raise + + return module + + +def _install_meta_finder() -> None: + """Install the meta path finder if it's not already installed.""" + finder_class = ScyJavaTypesMetaFinder + + # Check if our finder is already installed + for finder in sys.meta_path: + if isinstance(finder, finder_class): + return + + # Install our finder at the beginning of meta_path + sys.meta_path.insert(0, finder_class()) + + +def _uninstall_meta_finder() -> None: + """Remove the meta path finder.""" + finder_class = ScyJavaTypesMetaFinder + + # Remove all instances of our finder + sys.meta_path[:] = [ + finder for finder in sys.meta_path if not isinstance(finder, finder_class) + ] + + +# Install the meta finder when this module is imported +_install_meta_finder() + + +def get_registered_generator() -> Optional[Callable[[str], Optional[types.ModuleType]]]: + """ + Get the currently registered module generator function. + + Returns: + The registered generator function, or None if none is registered + """ + return _module_generator + + +def is_meta_finder_installed() -> bool: + """ + Check if the meta path finder is currently installed. + + Returns: + True if the finder is installed, False otherwise + """ + finder_class = ScyJavaTypesMetaFinder + return any(isinstance(finder, finder_class) for finder in sys.meta_path) + + +def clear_generated_modules() -> None: + """ + Clear all dynamically generated modules from sys.modules. + + This is useful for testing or when you want to regenerate modules + with a different generator function. + """ + base_package = __name__ + to_remove = [ + name for name in sys.modules.keys() + if name.startswith(base_package + ".") and name != base_package + ] + + for name in to_remove: + sys.modules.pop(name, None) + + +def list_generated_modules() -> list[str]: + """ + List all currently loaded dynamically generated modules. + + Returns: + A list of module names that were generated by this meta finder + """ + base_package = __name__ + generated = [] + + for name, module in sys.modules.items(): + if (name.startswith(base_package + ".") and + name != base_package and + hasattr(module, '__file__') and + module.__file__ and + (' Optional[types.ModuleType]: + """ + Example module generator function for testing. + + This would be replaced with your actual module generation logic. + + Args: + module_name: The full module name being imported + + Returns: + A dynamically generated module + """ + # Create a new module + module = types.ModuleType(module_name) + + # Add some example attributes based on the module name + parts = module_name.split(".") + if len(parts) > 2: # scyjava.types.XXX + class_name = parts[-1] + + # Add a dummy class with the same name as the module + dummy_class = type( + class_name, + (), + { + "__module__": module_name, + "__doc__": f"Dynamically generated class for {class_name}", + }, + ) + + setattr(module, class_name, dummy_class) + # Set __all__ as a regular attribute using setattr + setattr(module, '__all__', [class_name]) + + module.__file__ = f"" + module.__package__ = ".".join(parts[:-1]) if len(parts) > 1 else None + + return module + + +def _test_dynamic_import(): + """Test function to verify the meta finder works.""" + # Set up the example generator + set_module_generator(_example_module_generator) + + try: + # This should trigger our meta finder + # Type checker will complain since SomeTestClass doesn't exist statically + from scyjava.types import SomeTestClass # type: ignore + + print(f"Successfully imported: {SomeTestClass}") + print(f"Module name: {SomeTestClass.__name__}") + # Get the class from the module + if hasattr(SomeTestClass, 'SomeTestClass'): + test_class = getattr(SomeTestClass, 'SomeTestClass') + print(f"Class module: {test_class.__module__}") + return True + except ImportError as e: + print(f"Import failed: {e}") + return False + + +if __name__ == "__main__": + # Run test if this module is executed directly + _test_dynamic_import() diff --git a/test_meta_finder.py b/test_meta_finder.py new file mode 100644 index 0000000..48391b2 --- /dev/null +++ b/test_meta_finder.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +""" +Test script demonstrating the scyjava.types meta path finder. +""" + +import sys + +sys.path.insert(0, "src") + +import types +import scyjava.types + + +def java_class_generator(module_name: str) -> types.ModuleType: + """ + Example generator that creates Java-like classes dynamically. + + This is where you would implement your actual Java class introspection + and stub generation logic. + """ + print(f"šŸ”§ Generating module for: {module_name}") + + # Create the module + module = types.ModuleType(module_name) + module.__file__ = f"" + + # Extract class name from module path + parts = module_name.split(".") + if len(parts) > 2: # scyjava.types.ClassName + class_name = parts[-1] + + # Create a mock Java class + def __init__(self, *args, **kwargs): + self._java_args = args + self._java_kwargs = kwargs + print(f"šŸ“¦ Created {class_name} instance with args={args}, kwargs={kwargs}") + + def toString(self): + return f"{class_name}@{id(self):x}" + + # Create the class dynamically + java_class = type( + class_name, + (), + { + "__module__": module_name, + "__doc__": f"Dynamically generated Java class: {class_name}", + "__init__": __init__, + "toString": toString, + # Add some Java-like methods + "getClass": lambda self: type(self), + "equals": lambda self, other: self is other, + "hashCode": lambda self: hash(id(self)), + }, + ) + + # Add the class to the module + setattr(module, class_name, java_class) + setattr(module, "__all__", [class_name]) + + print(f"āœ… Generated class {class_name} in module {module_name}") + + return module + + +def main(): + print("šŸš€ Testing scyjava.types meta path finder") + print("=" * 50) + + # Register our generator + print("1. Registering module generator...") + scyjava.types.set_module_generator(java_class_generator) + + print("\n2. Testing dynamic imports...") + + # Test 1: Import a "Java" class + print("\nšŸ“„ Importing scyjava.types.ArrayList...") + import scyjava.types.ArrayList as arraylist_module + + ArrayList = arraylist_module.ArrayList + + # Create an instance + arr = ArrayList(10, "initial", capacity=100) + print(f"Created ArrayList: {arr.toString()}") + print(f"Class: {arr.getClass()}") + + # Test 2: Import another class + print("\nšŸ“„ Importing scyjava.types.HashMap...") + from scyjava.types import HashMap as hashmap_module + + HashMap = hashmap_module.HashMap + + map_obj = HashMap() + print(f"Created HashMap: {map_obj.toString()}") + + # Test 3: Show that imports are cached + print("\nšŸ“„ Re-importing ArrayList (should be cached)...") + import scyjava.types.ArrayList as arraylist_module2 + + print(f"Same module? {arraylist_module is arraylist_module2}") + + print("\nāœ… All tests passed!") + + # Show what's in sys.modules + print("\nšŸ“‹ Dynamic modules in sys.modules:") + for name, module in sys.modules.items(): + if name.startswith("scyjava.types.") and hasattr(module, "__file__"): + if " Date: Fri, 22 Aug 2025 21:35:08 -0400 Subject: [PATCH 24/29] working principle --- src/scyjava/_stubs/_genstubs.py | 2 +- src/scyjava/types/__init__.py | 402 ++++++++------------------------ 2 files changed, 100 insertions(+), 304 deletions(-) diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 3df24cb..7f95cc1 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -90,7 +90,7 @@ def generate_stubs( "stubgenj is not installed, but is required to generate java stubs. " "Please install it with `pip/conda install stubgenj`." ) from e - + print("GENERATE") import jpype startJVM = jpype.startJVM diff --git a/src/scyjava/types/__init__.py b/src/scyjava/types/__init__.py index ac25d2c..a3111c7 100644 --- a/src/scyjava/types/__init__.py +++ b/src/scyjava/types/__init__.py @@ -1,358 +1,154 @@ -""" -Dynamic type-safe imports for scyjava types with lazy initialization. +"""Dynamic type-safe imports for scyjava types with lazy initialization. This module provides a meta path finder that intercepts imports from scyjava.types and dynamically generates the requested modules at import time. -Usage: - 1. Set up a module generator function: - - def my_generator(module_name: str) -> types.ModuleType: - # Create and return a module based on module_name - module = types.ModuleType(module_name) - # ... populate module with classes/functions ... - return module - - 2. Register the generator: - - scyjava.types.set_module_generator(my_generator) - - 3. Import dynamically generated modules: - - import scyjava.types.SomeClass as some_class_module - SomeClass = some_class_module.SomeClass - - # Or use from import: - from scyjava.types import SomeClass as some_class_module - SomeClass = some_class_module.SomeClass - The generator function will be called at import time with the full module name (e.g., "scyjava.types.SomeClass") and should return a populated module. - -API Functions: - - set_module_generator(func): Register a module generator function - - get_registered_generator(): Get the currently registered generator - - is_meta_finder_installed(): Check if the meta finder is active - - clear_generated_modules(): Remove all generated modules from cache - - list_generated_modules(): List all currently loaded generated modules - -Example: - >>> import scyjava.types - >>> - >>> def java_stub_generator(module_name): - ... import types - ... module = types.ModuleType(module_name) - ... class_name = module_name.split('.')[-1] - ... # Generate Java class stub here... - ... java_class = create_java_stub(class_name) - ... setattr(module, class_name, java_class) - ... return module - ... - >>> scyjava.types.set_module_generator(java_stub_generator) - >>> - >>> # This will call java_stub_generator("scyjava.types.ArrayList") - >>> import scyjava.types.ArrayList as al_module - >>> ArrayList = al_module.ArrayList """ +from __future__ import annotations + import importlib.util import sys +import threading import types -from typing import Any, Callable, Optional, Sequence - -# The function that will be called to generate modules at import time -_module_generator: Optional[Callable[[str], Optional[types.ModuleType]]] = None +from ast import mod +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import SourceFileLoader +from pathlib import Path +from typing import TYPE_CHECKING +import scyjava +from scyjava._stubs import generate_stubs -def set_module_generator(func: Callable[[str], Optional[types.ModuleType]]) -> None: - """ - Set the function that will be called to generate modules at import time. +if TYPE_CHECKING: + from collections.abc import Sequence + from importlib.machinery import ModuleSpec - Args: - func: A callable that takes a module name (str) and returns a module - or None if the module cannot be generated. - """ - global _module_generator - _module_generator = func +_STUBS_LOCK = threading.Lock() +TYPES_DIR = Path(__file__).parent -class ScyJavaTypesMetaFinder: - """ - A meta path finder that intercepts imports from scyjava.types and - dynamically generates modules at import time. - """ - @classmethod - def _get_base_package_name(cls) -> str: - """Get the base package name for portability.""" - # Use __name__ for portability - this will be 'scyjava.types' - return __name__ +class ScyJavaTypesMetaFinder(MetaPathFinder): + """Meta path finder for scyjava.types that delegates to our loader.""" def find_spec( self, fullname: str, - path: Optional[Sequence[str]] = None, - target: Optional[types.ModuleType] = None, - ): - """ - Find a module spec for dynamic module generation. - - Args: - fullname: The fully qualified name of the module being imported - path: The search path (unused) - target: The target module (unused) - - Returns: - A ModuleSpec if this finder should handle the import, None otherwise - """ - base_package = self._get_base_package_name() - - # Only handle imports that start with our base package - if not fullname.startswith(base_package + "."): + path: Sequence[str] | None, + target: types.ModuleType | None = None, + /, + ) -> ModuleSpec | None: + """Return a spec for names under scyjava.types (except the base).""" + base_package = __name__ + + if not fullname.startswith(base_package + ".") or fullname == base_package: return None - # Don't handle the base package itself - if fullname == base_package: - return None - - # Create a module spec with our custom loader - spec = importlib.util.spec_from_loader( - fullname, ScyJavaTypesLoader(fullname), origin="dynamic" + return importlib.util.spec_from_loader( + fullname, + ScyJavaTypesLoader(fullname), + origin="dynamic", ) - return spec -class ScyJavaTypesLoader: - """ - A module loader that calls the registered module generator function - to create modules dynamically at import time. - """ +class ScyJavaTypesLoader(Loader): + """Loader that lazily generates stubs and loads/synthesizes modules.""" - def __init__(self, fullname: str): + def __init__(self, fullname: str) -> None: self.fullname = fullname - def create_module(self, spec) -> Optional[types.ModuleType]: - """ - Create a module by calling the registered generator function. + def create_module(self, spec: ModuleSpec) -> types.ModuleType | None: + """Load an existing module/package or lazily generate stubs then load.""" + pkg_dir, pkg_init, mod_file = _paths_for(spec.name, TYPES_DIR) - Args: - spec: The module spec + def _load_module() -> types.ModuleType | None: + # Fast paths: concrete module file or package present + if pkg_init.exists() or mod_file.exists(): + return _load_generated_module(spec.name, TYPES_DIR) + if pkg_dir.is_dir(): + return _namespace_package(spec, pkg_dir) + return None + + if module := _load_module(): + return module - Returns: - The generated module or None to use default module creation - """ - if _module_generator is None: - raise ImportError( - f"No module generator registered for {self.fullname}. " - f"Call scyjava.types.set_module_generator() first." - ) + # Nothing exists for this name: generate once under a lock + with _STUBS_LOCK: + # Re-check under the lock to avoid duplicate work + if not (pkg_init.exists() or mod_file.exists() or pkg_dir.exists()): + endpoints = ["org.scijava:parsington:3.1.0"] # TODO + generate_stubs(endpoints, output_dir=TYPES_DIR) - # Call the registered function to generate the module - module = _module_generator(self.fullname) - if module is None: - raise ImportError( - f"Module generator failed to create module: {self.fullname}" - ) + # Retry after generation (or if another thread created it) + if module := _load_module(): + return module - return module + raise ImportError(f"Generated module not found: {spec.name} under {pkg_dir}") def exec_module(self, module: types.ModuleType) -> None: - """ - Execute the module. Since our modules are generated dynamically, - there's nothing to execute here. - - Args: - module: The module to execute - """ - # Module is already fully initialized by the generator function pass - - def load_module(self, fullname: str) -> types.ModuleType: - """ - Load a module (deprecated method, kept for compatibility). - - Args: - fullname: The fully qualified module name - - Returns: - The loaded module - """ - # Check if module is already in sys.modules - if fullname in sys.modules: - return sys.modules[fullname] - - # Create module spec and load it - spec = importlib.util.spec_from_loader(fullname, self, origin="dynamic") - if spec is None: - raise ImportError(f"Failed to create spec for module: {fullname}") - - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Failed to create module: {fullname}") - - # Add to sys.modules before exec - sys.modules[fullname] = module - - try: - if spec.loader: - spec.loader.exec_module(module) - except Exception: - # Remove from sys.modules if exec fails - sys.modules.pop(fullname, None) - raise - - return module - - -def _install_meta_finder() -> None: - """Install the meta path finder if it's not already installed.""" - finder_class = ScyJavaTypesMetaFinder - # Check if our finder is already installed - for finder in sys.meta_path: - if isinstance(finder, finder_class): - return - # Install our finder at the beginning of meta_path - sys.meta_path.insert(0, finder_class()) +def _paths_for(fullname: str, out_dir: Path) -> tuple[Path, Path, Path]: + """Return (pkg_dir, pkg_init, mod_file) for a scyjava.types.* fullname.""" + rel = fullname.split("scyjava.types.", 1)[1] + pkg_dir = out_dir / rel.replace(".", "/") + pkg_init = pkg_dir / "__init__.py" + mod_file = out_dir / (rel.replace(".", "/") + ".py") + return pkg_dir, pkg_init, mod_file -def _uninstall_meta_finder() -> None: - """Remove the meta path finder.""" - finder_class = ScyJavaTypesMetaFinder +def _namespace_package(spec: ModuleSpec, pkg_dir: Path) -> types.ModuleType: + """Create a simple package module pointing at pkg_dir. - # Remove all instances of our finder - sys.meta_path[:] = [ - finder for finder in sys.meta_path if not isinstance(finder, finder_class) - ] - - -# Install the meta finder when this module is imported -_install_meta_finder() - - -def get_registered_generator() -> Optional[Callable[[str], Optional[types.ModuleType]]]: - """ - Get the currently registered module generator function. - - Returns: - The registered generator function, or None if none is registered - """ - return _module_generator - - -def is_meta_finder_installed() -> bool: - """ - Check if the meta path finder is currently installed. - - Returns: - True if the finder is installed, False otherwise + This fills the role of a namespace package, (a folder with no __init__.py). """ - finder_class = ScyJavaTypesMetaFinder - return any(isinstance(finder, finder_class) for finder in sys.meta_path) + module = types.ModuleType(spec.name) + module.__package__ = spec.name + module.__path__ = [str(pkg_dir)] + module.__spec__ = spec + return module -def clear_generated_modules() -> None: - """ - Clear all dynamically generated modules from sys.modules. - - This is useful for testing or when you want to regenerate modules - with a different generator function. - """ - base_package = __name__ - to_remove = [ - name for name in sys.modules.keys() - if name.startswith(base_package + ".") and name != base_package - ] - - for name in to_remove: - sys.modules.pop(name, None) +def _load_generated_module(fullname: str, out_dir: Path) -> types.ModuleType: + """Load a just-generated module/package from disk and return it.""" + _, pkg_init, mod_file = _paths_for(fullname, out_dir) + path = pkg_init if pkg_init.exists() else mod_file + if not path.exists(): + raise ImportError(f"Generated module not found: {fullname} at {path}") + loader = SourceFileLoader(fullname, str(path)) + spec = importlib.util.spec_from_loader(fullname, loader, origin=str(path)) + if spec is None or spec.loader is None: + raise ImportError(f"Failed to build spec for: {fullname}") -def list_generated_modules() -> list[str]: - """ - List all currently loaded dynamically generated modules. - - Returns: - A list of module names that were generated by this meta finder - """ - base_package = __name__ - generated = [] - - for name, module in sys.modules.items(): - if (name.startswith(base_package + ".") and - name != base_package and - hasattr(module, '__file__') and - module.__file__ and - (' Optional[types.ModuleType]: - """ - Example module generator function for testing. - - This would be replaced with your actual module generation logic. - - Args: - module_name: The full module name being imported - - Returns: - A dynamically generated module - """ - # Create a new module - module = types.ModuleType(module_name) - - # Add some example attributes based on the module name - parts = module_name.split(".") - if len(parts) > 2: # scyjava.types.XXX - class_name = parts[-1] - - # Add a dummy class with the same name as the module - dummy_class = type( - class_name, - (), - { - "__module__": module_name, - "__doc__": f"Dynamically generated class for {class_name}", - }, - ) + spec.has_location = True # populate __file__ + sys.modules[fullname] = module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module - setattr(module, class_name, dummy_class) - # Set __all__ as a regular attribute using setattr - setattr(module, '__all__', [class_name]) - module.__file__ = f"" - module.__package__ = ".".join(parts[:-1]) if len(parts) > 1 else None +# ----------------------------------------------------------- - return module +def _install_meta_finder() -> None: + for finder in sys.meta_path: + if isinstance(finder, ScyJavaTypesMetaFinder): + return -def _test_dynamic_import(): - """Test function to verify the meta finder works.""" - # Set up the example generator - set_module_generator(_example_module_generator) + sys.meta_path.insert(0, ScyJavaTypesMetaFinder()) - try: - # This should trigger our meta finder - # Type checker will complain since SomeTestClass doesn't exist statically - from scyjava.types import SomeTestClass # type: ignore - print(f"Successfully imported: {SomeTestClass}") - print(f"Module name: {SomeTestClass.__name__}") - # Get the class from the module - if hasattr(SomeTestClass, 'SomeTestClass'): - test_class = getattr(SomeTestClass, 'SomeTestClass') - print(f"Class module: {test_class.__module__}") - return True - except ImportError as e: - print(f"Import failed: {e}") - return False +def uninstall_meta_finder() -> None: + """Uninstall the ScyJavaTypesMetaFinder from sys.meta_path.""" + sys.meta_path[:] = [ + finder + for finder in sys.meta_path + if not isinstance(finder, ScyJavaTypesMetaFinder) + ] -if __name__ == "__main__": - # Run test if this module is executed directly - _test_dynamic_import() +_install_meta_finder() From 76d9bfa0951fe0641dffdfcd2505b63fc1a68355 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 22 Aug 2025 21:37:07 -0400 Subject: [PATCH 25/29] remove readme --- METAFINDER_README.md | 81 -------------------------------------------- 1 file changed, 81 deletions(-) delete mode 100644 METAFINDER_README.md diff --git a/METAFINDER_README.md b/METAFINDER_README.md deleted file mode 100644 index a147b0d..0000000 --- a/METAFINDER_README.md +++ /dev/null @@ -1,81 +0,0 @@ -# scyjava.types Meta Path Finder Infrastructure - -This module provides a complete meta path finder infrastructure for `scyjava.types` that: - -## Core Features - -1. **Intercepts imports** from `scyjava.types.XXX` using a custom meta path finder -2. **Calls a user-defined function** at import time to generate the module -3. **Executes the import** after the module is generated -4. **Uses `__name__` for portability** - works regardless of where the module is located - -## Key Components - -### ScyJavaTypesMetaFinder -- Implements the `find_spec()` method required by Python's import system -- Only handles imports that start with `scyjava.types.` -- Creates a `ModuleSpec` with our custom loader - -### ScyJavaTypesLoader -- Implements `create_module()` and `exec_module()` for the new import system -- Also implements `load_module()` for backward compatibility -- Calls the registered generator function to create modules - -### Module Generator Function -- User-provided function that receives the full module name -- Must return a `types.ModuleType` instance -- Called exactly once per unique import (modules are cached) - -## API Functions - -- `set_module_generator(func)` - Register your module generation function -- `get_registered_generator()` - Get the currently registered function -- `is_meta_finder_installed()` - Check if the meta finder is active -- `clear_generated_modules()` - Remove generated modules from cache -- `list_generated_modules()` - List all currently loaded generated modules - -## Usage Pattern - -```python -import scyjava.types - -def my_generator(module_name: str) -> types.ModuleType: - # module_name will be something like "scyjava.types.ArrayList" - module = types.ModuleType(module_name) - - # Extract class name from the module path - class_name = module_name.split('.')[-1] # "ArrayList" - - # Generate your class/content here - generated_class = create_java_stub(class_name) - setattr(module, class_name, generated_class) - - module.__file__ = f"" - return module - -# Register the generator -scyjava.types.set_module_generator(my_generator) - -# Now imports will trigger the generator -import scyjava.types.ArrayList as al_module -ArrayList = al_module.ArrayList - -# Or using from-import -from scyjava.types import HashMap as hm_module -HashMap = hm_module.HashMap -``` - -## Error Handling - -- If no generator is registered, imports raise `ImportError` -- If the generator returns `None`, imports raise `ImportError` -- Generator exceptions propagate to the import statement - -## Caching - -- Modules are automatically cached in `sys.modules` -- Subsequent imports of the same module return the cached version -- Use `clear_generated_modules()` to force regeneration - -This infrastructure provides the foundation for dynamic Java class stub generation -at import time, enabling type-safe imports with lazy initialization. From c57949089d1d1c2905dbfdb75556b340f42ecf4c Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 22 Aug 2025 21:37:13 -0400 Subject: [PATCH 26/29] remove x --- x.py | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 x.py diff --git a/x.py b/x.py deleted file mode 100644 index ff9849c..0000000 --- a/x.py +++ /dev/null @@ -1,4 +0,0 @@ -from scyjava.types.org.scijava.parsington import Operator - - -print(Operator) From 9e33f2ed350973b4e657634290e7d75824d635e4 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Sat, 23 Aug 2025 09:07:03 -0400 Subject: [PATCH 27/29] refactor: delay import of pandas until needed --- src/scyjava/_convert.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/scyjava/_convert.py b/src/scyjava/_convert.py index af1583b..c73f8b7 100644 --- a/src/scyjava/_convert.py +++ b/src/scyjava/_convert.py @@ -7,8 +7,9 @@ import logging import math from bisect import insort +from importlib.util import find_spec from pathlib import Path -from typing import Any, Callable, Dict, List, NamedTuple +from typing import Any, Callable, Dict, List, NamedTuple, reveal_type from jpype import JBoolean, JByte, JChar, JDouble, JFloat, JInt, JLong, JShort @@ -677,7 +678,7 @@ def _stock_py_converters() -> List: priority=Priority.VERY_LOW, ), ] - if _import_pandas(required=False): + if find_spec("pandas"): converters.append( Converter( name="org.scijava.table.Table -> pandas.DataFrame", @@ -716,7 +717,7 @@ def _stock_py_converters() -> List: ), ] ) - if _import_numpy(required=False): + if find_spec("numpy"): converters.append( Converter( name="primitive array -> numpy.ndarray", @@ -803,16 +804,15 @@ def _jarray_shape(jarr): return shape -def _import_numpy(required=True): +def _import_numpy(): try: import numpy as np return np except ImportError as e: - if required: - msg = "The NumPy library is missing (https://numpy.org/). " - msg += "Please install it before using this function." - raise RuntimeError(msg) from e + msg = "The NumPy library is missing (https://numpy.org/). " + msg += "Please install it before using this function." + raise RuntimeError(msg) from e ###################################### @@ -838,16 +838,15 @@ def _convert_table(obj: Any): return None -def _import_pandas(required=True): +def _import_pandas(): try: import pandas as pd return pd except ImportError as e: - if required: - msg = "The Pandas library is missing (http://pandas.pydata.org/). " - msg += "Please install it before using this function." - raise RuntimeError(msg) from e + msg = "The Pandas library is missing (http://pandas.pydata.org/). " + msg += "Please install it before using this function." + raise RuntimeError(msg) from e def _table_to_pandas(table): From 6f89935091be21f25af75214d6d28de1efcd68b2 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Sat, 23 Aug 2025 09:10:47 -0400 Subject: [PATCH 28/29] remove reveal type --- src/scyjava/_convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scyjava/_convert.py b/src/scyjava/_convert.py index c73f8b7..4a04f27 100644 --- a/src/scyjava/_convert.py +++ b/src/scyjava/_convert.py @@ -9,7 +9,7 @@ from bisect import insort from importlib.util import find_spec from pathlib import Path -from typing import Any, Callable, Dict, List, NamedTuple, reveal_type +from typing import Any, Callable, Dict, List, NamedTuple from jpype import JBoolean, JByte, JChar, JDouble, JFloat, JInt, JLong, JShort From 93e4e51ea184d2e9888b2a1bb71a1f1aff3a8f06 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Sat, 23 Aug 2025 13:51:17 -0400 Subject: [PATCH 29/29] cleaner --- src/scyjava/_stubs/__init__.py | 2 +- src/scyjava/_stubs/_genstubs.py | 35 ++++--- src/scyjava/py.typed | 0 src/scyjava/types/__init__.py | 160 ++++++++++++-------------------- tests/test_stubgen.py | 15 ++- 5 files changed, 87 insertions(+), 125 deletions(-) create mode 100644 src/scyjava/py.typed diff --git a/src/scyjava/_stubs/__init__.py b/src/scyjava/_stubs/__init__.py index d6a5e7c..ccfc0c7 100644 --- a/src/scyjava/_stubs/__init__.py +++ b/src/scyjava/_stubs/__init__.py @@ -1,4 +1,4 @@ from ._dynamic_import import setup_java_imports from ._genstubs import generate_stubs -__all__ = ["setup_java_imports", "generate_stubs"] +__all__ = ["generate_stubs", "setup_java_imports"] diff --git a/src/scyjava/_stubs/_genstubs.py b/src/scyjava/_stubs/_genstubs.py index 7f95cc1..e84ff12 100644 --- a/src/scyjava/_stubs/_genstubs.py +++ b/src/scyjava/_stubs/_genstubs.py @@ -65,8 +65,8 @@ def generate_stubs( called with the `convertStrings` argument set to True or False. By setting this `convert_strings` argument to true, the type stubs will be generated as if `convertStrings` is set to True: that is, all string types will be listed as - `str` rather than `java.lang.String | str`. This is a safer default (as `str`) - is a subtype of `java.lang.String`), but may lead to type errors in some cases. + `str` rather than `java.lang.String | str`. This is a safer default (as `str` + is a base of `java.lang.String`), but may lead to type errors in some cases. include_javadoc : bool, optional Whether to include Javadoc in the generated stubs. Defaults to True. add_runtime_imports : bool, optional @@ -90,9 +90,13 @@ def generate_stubs( "stubgenj is not installed, but is required to generate java stubs. " "Please install it with `pip/conda install stubgenj`." ) from e - print("GENERATE") import jpype + # if jpype.isJVMStarted(): + # raise RuntimeError( + # "Generating type stubs after the JVM has started is not supported." + # ) + startJVM = jpype.startJVM scyjava.config.endpoints.extend(endpoints) @@ -117,23 +121,24 @@ def _patched_start(*args: Any, **kwargs: Any) -> None: logger.info(f"Generating stubs for: {prefixes}") logger.info(f"Writing stubs to: {output_dir}") - metapath = sys.meta_path + metapath = sys.meta_path.copy() try: import jpype.imports jmodules = [import_module(prefix) for prefix in prefixes] + + stubgenj.generateJavaStubs( + jmodules, + useStubsSuffix=False, + outputDir=str(output_dir), + jpypeJPackageStubs=False, + includeJavadoc=include_javadoc, + ) + finally: - # remove the jpype.imports magic from the import system - # if it wasn't there to begin with - sys.meta_path = metapath - - stubgenj.generateJavaStubs( - jmodules, - useStubsSuffix=False, - outputDir=str(output_dir), - jpypeJPackageStubs=False, - includeJavadoc=include_javadoc, - ) + # restore sys.metapath + # (remove the jpype.imports magic if it wasn't there to begin with) + sys.meta_path[:] = metapath output_dir = Path(output_dir) if add_runtime_imports: diff --git a/src/scyjava/py.typed b/src/scyjava/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/scyjava/types/__init__.py b/src/scyjava/types/__init__.py index a3111c7..0d69bff 100644 --- a/src/scyjava/types/__init__.py +++ b/src/scyjava/types/__init__.py @@ -9,136 +9,96 @@ from __future__ import annotations -import importlib.util +import os import sys import threading -import types -from ast import mod -from importlib.abc import Loader, MetaPathFinder -from importlib.machinery import SourceFileLoader from pathlib import Path from typing import TYPE_CHECKING -import scyjava -from scyjava._stubs import generate_stubs - if TYPE_CHECKING: + import types from collections.abc import Sequence from importlib.machinery import ModuleSpec +# where generated stubs should land (defaults to this dir: `scyjava.types`) +STUBS_DIR = os.getenv("SCYJAVA_STUBS_DIR", str(Path(__file__).parent)) +# namespace under which generated stubs will be placed +STUBS_NAMESPACE = __name__ +# module lock to prevent concurrent stub generation _STUBS_LOCK = threading.Lock() -TYPES_DIR = Path(__file__).parent -class ScyJavaTypesMetaFinder(MetaPathFinder): - """Meta path finder for scyjava.types that delegates to our loader.""" +class ScyJavaTypesMetaFinder: + """Meta path finder for scyjava.types that generates stubs on demand.""" def find_spec( self, fullname: str, path: Sequence[str] | None, target: types.ModuleType | None = None, - /, ) -> ModuleSpec | None: """Return a spec for names under scyjava.types (except the base).""" - base_package = __name__ - - if not fullname.startswith(base_package + ".") or fullname == base_package: - return None - - return importlib.util.spec_from_loader( - fullname, - ScyJavaTypesLoader(fullname), - origin="dynamic", - ) - - -class ScyJavaTypesLoader(Loader): - """Loader that lazily generates stubs and loads/synthesizes modules.""" - - def __init__(self, fullname: str) -> None: - self.fullname = fullname - - def create_module(self, spec: ModuleSpec) -> types.ModuleType | None: - """Load an existing module/package or lazily generate stubs then load.""" - pkg_dir, pkg_init, mod_file = _paths_for(spec.name, TYPES_DIR) - - def _load_module() -> types.ModuleType | None: - # Fast paths: concrete module file or package present - if pkg_init.exists() or mod_file.exists(): - return _load_generated_module(spec.name, TYPES_DIR) - if pkg_dir.is_dir(): - return _namespace_package(spec, pkg_dir) - return None - - if module := _load_module(): - return module - - # Nothing exists for this name: generate once under a lock - with _STUBS_LOCK: - # Re-check under the lock to avoid duplicate work - if not (pkg_init.exists() or mod_file.exists() or pkg_dir.exists()): - endpoints = ["org.scijava:parsington:3.1.0"] # TODO - generate_stubs(endpoints, output_dir=TYPES_DIR) - - # Retry after generation (or if another thread created it) - if module := _load_module(): - return module + # if this is an import from this module ('scyjava.types.') + # check if the module exists, and if not, call generation routines + if fullname.startswith(f"{__name__}."): + with _STUBS_LOCK: + # check if the spec already exists + # under the module lock to avoid duplicate work + if not _find_spec(fullname, path, target, skip=self): + _generate_stubs() - raise ImportError(f"Generated module not found: {spec.name} under {pkg_dir}") + return None - def exec_module(self, module: types.ModuleType) -> None: - pass +def _generate_stubs() -> None: + """Install stubs for all endpoints detected in `scyjava.config`. -def _paths_for(fullname: str, out_dir: Path) -> tuple[Path, Path, Path]: - """Return (pkg_dir, pkg_init, mod_file) for a scyjava.types.* fullname.""" - rel = fullname.split("scyjava.types.", 1)[1] - pkg_dir = out_dir / rel.replace(".", "/") - pkg_init = pkg_dir / "__init__.py" - mod_file = out_dir / (rel.replace(".", "/") + ".py") - return pkg_dir, pkg_init, mod_file - - -def _namespace_package(spec: ModuleSpec, pkg_dir: Path) -> types.ModuleType: - """Create a simple package module pointing at pkg_dir. - - This fills the role of a namespace package, (a folder with no __init__.py). + This could be expanded to include additional endpoints detected in, for example, + python entry-points discovered in packages in the environment. """ - module = types.ModuleType(spec.name) - module.__package__ = spec.name - module.__path__ = [str(pkg_dir)] - module.__spec__ = spec - return module - - -def _load_generated_module(fullname: str, out_dir: Path) -> types.ModuleType: - """Load a just-generated module/package from disk and return it.""" - _, pkg_init, mod_file = _paths_for(fullname, out_dir) - path = pkg_init if pkg_init.exists() else mod_file - if not path.exists(): - raise ImportError(f"Generated module not found: {fullname} at {path}") - - loader = SourceFileLoader(fullname, str(path)) - spec = importlib.util.spec_from_loader(fullname, loader, origin=str(path)) - if spec is None or spec.loader is None: - raise ImportError(f"Failed to build spec for: {fullname}") + from scyjava import config + from scyjava._stubs import generate_stubs + + generate_stubs( + config.endpoints, + output_dir=STUBS_DIR, + add_runtime_imports=True, + remove_namespace_only_stubs=True, + ) + + +def _find_spec( + fullname: str, + path: Sequence[str] | None, + target: types.ModuleType | None = None, + skip: object | None = None, +) -> ModuleSpec | None: + """Find a module spec, skipping finder `skip` to avoid recursion.""" + # if the module is already loaded and has a spec, return it + if module := sys.modules.get(fullname): + try: + if module.__spec__ is not None: + return module.__spec__ + except AttributeError: + pass - spec.has_location = True # populate __file__ - sys.modules[fullname] = module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -# ----------------------------------------------------------- + for finder in sys.meta_path: + if finder is not skip: + try: + spec = finder.find_spec(fullname, path, target) + except AttributeError: + continue + else: + if spec is not None: + return spec + return None def _install_meta_finder() -> None: - for finder in sys.meta_path: - if isinstance(finder, ScyJavaTypesMetaFinder): - return - + """Install the ScyJavaTypesMetaFinder into sys.meta_path if not already there.""" + if any(isinstance(finder, ScyJavaTypesMetaFinder) for finder in sys.meta_path): + return sys.meta_path.insert(0, ScyJavaTypesMetaFinder()) diff --git a/tests/test_stubgen.py b/tests/test_stubgen.py index ad4a74a..31b7ea7 100644 --- a/tests/test_stubgen.py +++ b/tests/test_stubgen.py @@ -14,10 +14,11 @@ from pathlib import Path -@pytest.mark.skipif( - scyjava.config.mode != scyjava.config.Mode.JPYPE, - reason="Stubgen not supported in JEP", -) +JEP_MODE = scyjava.config.mode != scyjava.config.Mode.JPYPE +skip_if_jep = pytest.mark.skipif(JEP_MODE, reason="Stubgen not supported in JEP") + + +@skip_if_jep def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: # run the stubgen command as if it was run from the command line monkeypatch.setattr( @@ -32,10 +33,6 @@ def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: ) _cli.main() - # remove the `jpype.imports` magic from the import system if present - mp = [x for x in sys.meta_path if not isinstance(x, jpype.imports._JImportLoader)] - monkeypatch.setattr(sys, "meta_path", mp) - # add tmp_path to the import path monkeypatch.setattr(sys, "path", [str(tmp_path)]) @@ -43,8 +40,8 @@ def test_stubgen(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: sys.modules.pop("org", None) sys.modules.pop("org.scijava", None) sys.modules.pop("org.scijava.parsington", None) - # make sure the stubgen command works and that we can now impmort stuff + # make sure the stubgen command works and that we can now import stuff with patch.object(scyjava._jvm, "start_jvm") as mock_start_jvm: from org.scijava.parsington import Function