diff --git a/src/stubber/commands/clone_cmd.py b/src/stubber/commands/clone_cmd.py index c8a4201c..43a14839 100644 --- a/src/stubber/commands/clone_cmd.py +++ b/src/stubber/commands/clone_cmd.py @@ -34,6 +34,13 @@ is_flag=True, help="Also clone the micropython-stubs repo", ) +@click.option( + "--typeshed/--no-typeshed", + "typeshed", + default=False, + is_flag=True, + help="Also clone the typeshed repo", +) @click.option( "--mpy-repo", default="https://github.com/micropython/micropython.git", @@ -47,6 +54,7 @@ def cli_clone( path: Union[str, Path], stubs: bool = False, + typeshed: bool = False, mpy_repo: str = "https://github.com/micropython/micropython.git", mpy_lib_repo: str = "https://github.com/micropython/micropython-lib.git", ): @@ -66,11 +74,13 @@ def cli_clone( mpy_path = CONFIG.mpy_path mpy_lib_path = CONFIG.mpy_lib_path mpy_stubs_path = CONFIG.mpy_stubs_path + typeshed_path = CONFIG.typeshed_path else: # repos are relative to provided path mpy_path = dest_path / "micropython" mpy_lib_path = dest_path / "micropython-lib" mpy_stubs_path = dest_path / "micropython-stubs" + typeshed_path = dest_path / "typeshed" repos: List[Tuple[Path, str, str]] = [ (mpy_path, mpy_repo, "master"), @@ -78,6 +88,8 @@ def cli_clone( ] if stubs: repos.append((mpy_stubs_path, "https://github.com/josverl/micropython-stubs.git", "main")) + if typeshed: + repos.append((typeshed_path, "https://github.com/python/typeshed.git", "main")) for _path, remote, branch in repos: log.info(f"Cloning {remote} branch {branch} to {_path}") diff --git a/src/stubber/commands/stdlib_stubs_cmd.py b/src/stubber/commands/stdlib_stubs_cmd.py new file mode 100644 index 00000000..414ed1e0 --- /dev/null +++ b/src/stubber/commands/stdlib_stubs_cmd.py @@ -0,0 +1,489 @@ +""" +Command line interface to build micropython-stdlib-stubs package. + +This command integrates the functionality of building stdlib stubs from typeshed +and merging them with MicroPython documentation stubs. +""" + +import shutil +import subprocess +from pathlib import Path +from typing import Optional + +import rich_click as click +from mpflash.logger import log +from mpflash.versions import clean_version, get_stable_mp_version + +from stubber.codemod.enrich import enrich_folder +from stubber.commands.cli import stubber_cli +from stubber.modcat import STDLIB_ONLY_MODULES +from stubber.utils import do_post_processing +from stubber.utils.config import CONFIG + +# These modules will be kept in the stdlib folder +STDLIB_MODULES_TO_KEEP = list( + set(STDLIB_ONLY_MODULES) + | set( + [ + "_typeshed", + "asyncio", + "collections", + "sys", + "os", + "__future__", + "_ast", + "_codecs", + "_collections_abc", + "_decimal", + "abc", + "array", + "builtins", + "io", + "re", + "sys", + "types", + "typing_extensions", + "typing", + "tls", + "ssl", + "enum", + "sre_compile", + "sre_constants", + "sre_parse", + ] + ) +) + +# Try to limit the "overspeak" of python modules to the bare minimum +STDLIB_MODULES_TO_REMOVE = [ + "os/path.pyi", + "sys/_monitoring.pyi", + "asyncio/subprocess.pyi", + "asyncio/base_subprocess.pyi", + "asyncio/taskgroups.pyi", + "asyncio/windows_events.pyi", + "asyncio/windows_utils.pyi", + "json/decoder.pyi", + "json/encoder.pyi", + "json/tool.pyi", +] + +# Type ignore patterns +TYPE_IGNORES = [ + ("os", ["path = _path"]), + ("asyncio/taskgroups", [": Context", "from contextvars import Context"]), + ("asyncio/base_events", [": Context", "from contextvars import Context"]), + ("asyncio/base_futures", [": Context", "from contextvars import Context"]), + ("asyncio/events", [": Context", "from contextvars import Context"]), + ("asyncio/runners", [": Context", "from contextvars import Context"]), + ("_typeshed", ["Field[Any]"]), + ( + "builtins", + [ + ": int = -1", + ": int = 0", + " | None = None", + ": bool = True", + ": bool = False", + ': str | None = "', + ': str = "', + '| bytearray = b"', + ': bytes = b"', + ], + ), + ( + "collections", + [ + "deque[_T]", + "OrderedDict[", + "class deque(stdlib_deque):", + "class OrderedDict(stdlib_OrderedDict):", + ": _T, /", + "[_KT, _VT]", + ": _T,", + ": _KT,", + ": _VT,", + "-> _T:", + "-> _KT:", + "-> _VT:", + "Iterator[_T]", + "Iterator[_KT]", + ], + ), + ("io", ["from io import *"]), +] + +# Comment out some lines to hide CPython APIs +COMMENT_OUT_LINES = [ + ("asyncio", ["from .subprocess import *"]), + ( + "os", + [ + "from . import path as _path", + "def _exit(status: int) -> NoReturn: ...", + ], + ), +] + +# Change some lines to hide CPython APIs +CHANGE_LINES = [ + ( + "ssl", + [ + ("def create_default_context", "def __mpy_has_no_create_default_context"), + ("if sys.version_info < (3, 12):", "if True:"), + ], + ), + ( + "sys", + [ + ("def _getframe(", "def __mpy_has_no_getframe("), + ], + ), +] + + +def _extract_error_lines(text: str, max_lines: int = 10) -> str: + """Extract concise error lines from command output.""" + lines = [line.strip() for line in text.strip().split("\n") if line.strip()] + if len(lines) > max_lines: + lines = lines[-max_lines:] + return "\n".join(lines) + + +def update_stdlib_from_typeshed(dist_stdlib_path: Path, typeshed_path: Path) -> None: + """Update stdlib folder from typeshed repository.""" + log.info("Updating stdlib from typeshed") + + # Clear the stdlib folder + stdlib_path = dist_stdlib_path / "stdlib" + if stdlib_path.exists(): + shutil.rmtree(stdlib_path) + stdlib_path.mkdir(parents=True, exist_ok=True) + + # Copy modules from typeshed + typeshed_stdlib = typeshed_path / "stdlib" + if not typeshed_stdlib.exists(): + raise FileNotFoundError(f"Typeshed stdlib path not found: {typeshed_stdlib}") + + for module in STDLIB_MODULES_TO_KEEP: + src_path = typeshed_stdlib / module + if src_path.is_file(): + # Single file module + dst_path = stdlib_path / module + dst_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_path, dst_path) + log.debug(f"Copied {module}") + elif src_path.is_dir(): + # Package module + dst_path = stdlib_path / module + shutil.copytree(src_path, dst_path, dirs_exist_ok=True) + log.debug(f"Copied package {module}") + else: + # Try with .pyi extension + src_pyi = typeshed_stdlib / f"{module}.pyi" + if src_pyi.exists(): + dst_path = stdlib_path / f"{module}.pyi" + dst_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_pyi, dst_path) + log.debug(f"Copied {module}.pyi") + + # Remove unwanted modules + for module_path in STDLIB_MODULES_TO_REMOVE: + full_path = stdlib_path / module_path + if full_path.exists(): + full_path.unlink() + log.debug(f"Removed {module_path}") + + +def update_mpy_shed(reference_path: Path, dist_stdlib_path: Path) -> None: + """Update _mpy_shed stubs from reference folder.""" + log.info("Updating _mpy_shed") + src_mpy_shed = reference_path / "_mpy_shed" + dst_mpy_shed = dist_stdlib_path / "stdlib" / "_mpy_shed" + + if src_mpy_shed.exists(): + if dst_mpy_shed.exists(): + shutil.rmtree(dst_mpy_shed) + shutil.copytree(src_mpy_shed, dst_mpy_shed, dirs_exist_ok=True) + log.debug("Updated _mpy_shed") + + +def update_asyncio_manual(reference_path: Path, dist_stdlib_path: Path) -> None: + """Update manually maintained asyncio stubs.""" + log.info("Updating asyncio manual stubs") + src_asyncio = reference_path / "asyncio" + dst_asyncio = dist_stdlib_path / "stdlib" / "asyncio" + + if src_asyncio.exists(): + shutil.copytree(src_asyncio, dst_asyncio, dirs_exist_ok=True) + log.debug("Updated asyncio stubs") + + +def merge_docstubs_into_stdlib( + dist_stdlib_path: Path, + docstubs_path: Path, + boardstub_path: Path, +) -> None: + """Merge documentation stubs into stdlib.""" + log.info("Merging docstubs into stdlib") + stdlib_path = dist_stdlib_path / "stdlib" + + # Merge from docstubs + if docstubs_path.exists(): + result = enrich_folder( + stub_folder=stdlib_path, + docstub_folder=docstubs_path, + ) + log.info(f"Merged {result} files from docstubs") + + # Merge from board stubs + if boardstub_path.exists(): + result = enrich_folder( + stub_folder=stdlib_path, + docstub_folder=boardstub_path, + ) + log.info(f"Merged {result} files from board stubs") + + +def add_type_ignore(stdlib_path: Path) -> None: + """Add type ignore comments to reduce typechecker noise.""" + log.info("Adding type ignore comments") + + for module, patterns in TYPE_IGNORES: + module_file = stdlib_path / f"{module}.pyi" + if not module_file.exists(): + continue + + content = module_file.read_text(encoding="utf-8") + modified = False + + for pattern in patterns: + if pattern in content: + # Add # type: ignore comment at end of lines containing pattern + lines = content.split("\n") + new_lines = [] + for line in lines: + if pattern in line and "# type: ignore" not in line: + line = line.rstrip() + " # type: ignore" + modified = True + new_lines.append(line) + content = "\n".join(new_lines) + + if modified: + module_file.write_text(content, encoding="utf-8") + log.debug(f"Added type ignores to {module}") + + +def comment_out_lines(stdlib_path: Path) -> None: + """Comment out lines that cause issues.""" + log.info("Commenting out problematic lines") + + for module, patterns in COMMENT_OUT_LINES: + module_file = stdlib_path / f"{module}.pyi" + if not module_file.exists(): + continue + + content = module_file.read_text(encoding="utf-8") + modified = False + + for pattern in patterns: + if pattern in content: + content = content.replace(pattern, f"# {pattern}") + modified = True + + if modified: + module_file.write_text(content, encoding="utf-8") + log.debug(f"Commented out lines in {module}") + + +def change_lines(stdlib_path: Path) -> None: + """Change lines to hide CPython APIs.""" + log.info("Changing problematic lines") + + for module, replacements in CHANGE_LINES: + module_file = stdlib_path / f"{module}.pyi" + if not module_file.exists(): + continue + + content = module_file.read_text(encoding="utf-8") + modified = False + + for old_text, new_text in replacements: + if old_text in content: + content = content.replace(old_text, new_text) + modified = True + + if modified: + module_file.write_text(content, encoding="utf-8") + log.debug(f"Changed lines in {module}") + + +def update_typing_pyi(rootpath: Path, dist_stdlib_path: Path) -> None: + """Update typing.pyi with patches.""" + log.info("Updating typing.pyi") + # Placeholder for typing.pyi specific patches if needed + + +@stubber_cli.command(name="stdlib") +@click.option( + "--version", + "-v", + type=str, + help="Specify MicroPython version", + default=None, + show_default=True, +) +@click.option( + "--update/--no-update", + "-u", + help="Update stdlib from the typeshed repo.", + default=False, + show_default=True, +) +@click.option( + "--merge/--no-merge", + "-m", + help="Merge the docstubs into the stdlib.", + default=True, + show_default=True, +) +@click.option( + "--publish/--no-publish", + help="Publish the stdlib-stubs module.", + default=False, + show_default=True, +) +@click.option( + "--build/--no-build", + "-b", + help="Build the stdlib-stubs module.", + default=True, + show_default=True, +) +def cli_stdlib_stubs( + version: Optional[str] = None, + update: bool = False, + merge: bool = True, + build: bool = True, + publish: bool = False, +): + """ + Build the micropython-stdlib-stubs package. + + This command manages the creation of the stdlib-stubs package by: + - Updating from typeshed repository + - Merging with MicroPython documentation stubs + - Post-processing and formatting + - Building and optionally publishing the package + """ + # Determine version + if not version: + version = get_stable_mp_version() + + flat_version = clean_version(version, flat=True, drop_v=False) + log.info(f"Build micropython-stdlib-stubs for version: {version}") + + # Determine paths based on CONFIG + rootpath = CONFIG.stub_path.parent if CONFIG.stub_path else Path.cwd() + log.info(f"Using rootpath: {rootpath}") + + dist_stdlib_path = rootpath / "publish/micropython-stdlib-stubs" + docstubs_path = rootpath / f"stubs/micropython-{flat_version}-docstubs" + boardstub_path = rootpath / f"stubs/micropython-{flat_version}-esp32-ESP32_GENERIC" + typeshed_path = CONFIG.repo_path / CONFIG.typeshed_path + reference_path = rootpath / "reference" + + # Validate required paths + if not rootpath.exists(): + raise click.ClickException(f"Root path {rootpath} does not exist") + + # Create dist_stdlib_path if it doesn't exist + dist_stdlib_path.mkdir(parents=True, exist_ok=True) + + if update: + if not typeshed_path.exists(): + raise click.ClickException( + f"Typeshed path {typeshed_path} does not exist. Please clone it first using: stubber clone --typeshed" + ) + update_stdlib_from_typeshed(dist_stdlib_path, typeshed_path) + + # Always update _mpy_shed and asyncio + if reference_path.exists(): + update_mpy_shed(reference_path, dist_stdlib_path) + update_asyncio_manual(reference_path, dist_stdlib_path) + + if merge: + if not docstubs_path.exists(): + log.warning(f"Docstubs path {docstubs_path} does not exist, skipping merge") + else: + merge_docstubs_into_stdlib( + dist_stdlib_path=dist_stdlib_path, + docstubs_path=docstubs_path, + boardstub_path=boardstub_path if boardstub_path.exists() else None, + ) + + # Post-process the stubs + stdlib_path = dist_stdlib_path / "stdlib" + if stdlib_path.exists(): + do_post_processing([stdlib_path], stubgen=False, format=True, autoflake=True) + add_type_ignore(stdlib_path) + comment_out_lines(stdlib_path) + change_lines(stdlib_path) + + # Update last changed time + pyproject_path = dist_stdlib_path / "pyproject.toml" + if pyproject_path.exists(): + pyproject_path.touch() + + # Update typing.pyi + update_typing_pyi(rootpath, dist_stdlib_path) + + if build or publish: + if not (dist_stdlib_path / "pyproject.toml").exists(): + raise click.ClickException(f"No pyproject.toml found in {dist_stdlib_path}. Cannot build package.") + + try: + log.info("Building stdlib-stubs package...") + subprocess.check_call( + ["uv", "build", "--index-strategy", "unsafe-best-match", "--wheel"], + cwd=dist_stdlib_path, + ) + log.info("Build completed successfully") + except subprocess.CalledProcessError as e: + msg = _extract_error_lines(getattr(e, "stderr", "") or getattr(e, "output", "") or str(e)) + if msg: + raise click.ClickException(msg) from None + raise click.ClickException(f"Build failed with exit code {e.returncode}.") from None + except FileNotFoundError: + raise click.ClickException("uv not found. Please install uv: pip install uv") from None + + if publish: + try: + import keyring + + log.info(f"Publishing stdlib-stubs module... {version}") + publish_cmd = [ + "uv", + "publish", + "-u", + "__token__", + "-p", + keyring.get_password("stubber", "uv_pipy_token"), + ] + result = subprocess.run( + publish_cmd, + cwd=dist_stdlib_path, + text=True, + capture_output=True, + ) + if result.returncode != 0: + err = (result.stderr or result.stdout or "Publish failed").strip() + raise click.ClickException(err) + log.info("Published successfully") + except ImportError: + raise click.ClickException("keyring package not found. Please install: pip install keyring") from None + except Exception as e: + raise click.ClickException(f"Publish failed: {str(e)}") from None + + log.info("stdlib-stubs command completed successfully") diff --git a/src/stubber/stubber.py b/src/stubber/stubber.py index 7bc93f31..ca1bb7c5 100644 --- a/src/stubber/stubber.py +++ b/src/stubber/stubber.py @@ -13,6 +13,7 @@ from stubber.commands.get_frozen_cmd import cli_get_frozen from stubber.commands.merge_cmd import cli_merge_docstubs from stubber.commands.publish_cmd import cli_publish +from stubber.commands.stdlib_stubs_cmd import cli_stdlib_stubs from stubber.commands.stub_cmd import cli_stub from stubber.commands.switch_cmd import cli_switch from stubber.commands.variants_cmd import cli_variants @@ -32,6 +33,7 @@ stubber_cli.add_command(cli_enrich_folder) stubber_cli.add_command(cli_publish) stubber_cli.add_command(cli_merge_docstubs) + stubber_cli.add_command(cli_stdlib_stubs) stubber_cli.add_command(cli_variants) stubber_cli.add_command(cli_create_mcu_stubs) stubber_cli() diff --git a/src/stubber/utils/config.py b/src/stubber/utils/config.py index d2ef4a7e..9aee2855 100644 --- a/src/stubber/utils/config.py +++ b/src/stubber/utils/config.py @@ -1,7 +1,7 @@ """stubber configuration""" from pathlib import Path -from typing import List, Optional, Union +from typing import List, Optional from mpflash.logger import log from mpflash.versions import ( @@ -72,6 +72,14 @@ class StubberConfig(Config): ) "a Path to the micropython-stubs folder in the repos directory (or current directory)" + typeshed_path: Path = key( + key_name="typeshed-path", + cast=Path, + required=False, + default=Path("typeshed"), + ) + "a Path to the typeshed folder in the repos directory" + stable_version: str = key(key_name="stable-version", cast=str, required=False, default="1.20.0") "last published stable" @@ -92,7 +100,7 @@ class StubberConfig(Config): @property def repos(self) -> List[Path]: "return the repo paths" - return [self.mpy_path, self.mpy_lib_path, self.mpy_stubs_path] + return [self.mpy_path, self.mpy_lib_path, self.mpy_stubs_path, self.typeshed_path] @property def stub_path(self) -> Path: diff --git a/tests/commandline/stubber_cli_test.py b/tests/commandline/stubber_cli_test.py index cdd30884..17ae22af 100644 --- a/tests/commandline/stubber_cli_test.py +++ b/tests/commandline/stubber_cli_test.py @@ -59,7 +59,7 @@ def test_cmd_clone_path(mocker: MockerFixture, tmp_path: Path): m_clone = mocker.patch("stubber.commands.clone_cmd.git.clone", autospec=True, return_value=0) m_tag = mocker.patch("stubber.commands.clone_cmd.git.get_local_tag", autospec=True) - m_dir = mocker.patch("stubber.commands.clone_cmd.os.mkdir", autospec=True) # type: ignore + mocker.patch("stubber.commands.clone_cmd.os.mkdir", autospec=True) # type: ignore # now test with path specified result = runner.invoke(stubber.stubber_cli, ["clone", "--path", "foobar"]) @@ -77,6 +77,24 @@ def test_cmd_clone_path(mocker: MockerFixture, tmp_path: Path): assert m_tag.call_count >= 2 +@pytest.mark.mocked +def test_cmd_clone_typeshed(mocker: MockerFixture, tmp_path: Path): + """Test clone command with --typeshed option.""" + runner = CliRunner() + m_clone = mocker.patch("stubber.commands.clone_cmd.git.clone", autospec=True, return_value=0) + mocker.patch("stubber.commands.clone_cmd.git.get_local_tag", autospec=True) + + # Test with --typeshed flag + result = runner.invoke(stubber.stubber_cli, ["clone", "--typeshed"]) + assert result.exit_code == 0 + + # Should clone micropython, micropython-lib, and typeshed + assert m_clone.call_count >= 2 # At least mpy and mpy-lib + # Check if typeshed was cloned + typeshed_cloned = any(call[1].get("remote_repo") == "https://github.com/python/typeshed.git" for call in m_clone.call_args_list) + assert typeshed_cloned or m_clone.call_count >= 3 + + ########################################################################################## # switch ########################################################################################## diff --git a/tests/commandline/test_stdlib_stubs_cmd.py b/tests/commandline/test_stdlib_stubs_cmd.py new file mode 100644 index 00000000..74a479f1 --- /dev/null +++ b/tests/commandline/test_stdlib_stubs_cmd.py @@ -0,0 +1,554 @@ +"""Tests for the stdlib command.""" + +from pathlib import Path + +import pytest +from click.testing import CliRunner +from pytest_mock import MockerFixture + +import stubber.stubber as stubber + +# mark all tests +pytestmark = [pytest.mark.stubber, pytest.mark.cli] + + +########################################################################################## +# stdlib +########################################################################################## + + +@pytest.mark.mocked +def test_cmd_stdlib_stubs_help(): + """Test that the command shows help without errors.""" + runner = CliRunner() + result = runner.invoke(stubber.stubber_cli, ["stdlib", "--help"]) + assert result.exit_code == 0 + assert "Build the micropython-stdlib-stubs package" in result.output + + +@pytest.mark.mocked +def test_cmd_stdlib_stubs_no_build(mocker: MockerFixture, tmp_path: Path): + """Test stdlib command without building.""" + runner = CliRunner() + + # Mock get_stable_mp_version to return a fixed version + mocker.patch( + "stubber.commands.stdlib_stubs_cmd.get_stable_mp_version", + return_value="1.24.0", + ) + + # Mock Path operations and functions + m_update_mpy_shed = mocker.patch("stubber.commands.stdlib_stubs_cmd.update_mpy_shed") + m_update_asyncio = mocker.patch("stubber.commands.stdlib_stubs_cmd.update_asyncio_manual") + m_post_processing = mocker.patch("stubber.commands.stdlib_stubs_cmd.do_post_processing") + m_add_type_ignore = mocker.patch("stubber.commands.stdlib_stubs_cmd.add_type_ignore") + m_comment_out = mocker.patch("stubber.commands.stdlib_stubs_cmd.comment_out_lines") + m_change_lines = mocker.patch("stubber.commands.stdlib_stubs_cmd.change_lines") + m_update_typing = mocker.patch("stubber.commands.stdlib_stubs_cmd.update_typing_pyi") + + # Mock CONFIG by creating a fake config object + fake_config = mocker.MagicMock() + fake_config.stub_path = tmp_path / "stubs" + fake_config.repo_path = tmp_path / "repos" + fake_config.typeshed_path = Path("typeshed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.CONFIG", fake_config) + + # Create necessary directory structure + reference_path = tmp_path / "reference" + reference_path.mkdir(parents=True) + + stdlib_path = tmp_path / "publish" / "micropython-stdlib-stubs" / "stdlib" + stdlib_path.mkdir(parents=True, exist_ok=True) + + pyproject = tmp_path / "publish" / "micropython-stdlib-stubs" / "pyproject.toml" + pyproject.parent.mkdir(parents=True, exist_ok=True) + pyproject.write_text("[tool.poetry]\nname='test'\n") + + # Run command without building + result = runner.invoke( + stubber.stubber_cli, + ["stdlib", "--no-build", "--no-merge", "--no-update"], + ) + + # Command should succeed + assert result.exit_code == 0 + + # Verify the expected functions were called + assert m_update_mpy_shed.call_count == 1 + assert m_update_asyncio.call_count == 1 + assert m_post_processing.call_count == 1 + assert m_add_type_ignore.call_count == 1 + assert m_comment_out.call_count == 1 + assert m_change_lines.call_count == 1 + assert m_update_typing.call_count == 1 + + +@pytest.mark.mocked +def test_cmd_stdlib_stubs_with_merge(mocker: MockerFixture, tmp_path: Path): + """Test stdlib command with merge option.""" + runner = CliRunner() + + # Mock version function + mocker.patch( + "stubber.commands.stdlib_stubs_cmd.get_stable_mp_version", + return_value="1.24.0", + ) + + # Mock merge function + m_merge = mocker.patch("stubber.commands.stdlib_stubs_cmd.merge_docstubs_into_stdlib") + + # Mock other functions + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_mpy_shed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_asyncio_manual") + mocker.patch("stubber.commands.stdlib_stubs_cmd.do_post_processing") + mocker.patch("stubber.commands.stdlib_stubs_cmd.add_type_ignore") + mocker.patch("stubber.commands.stdlib_stubs_cmd.comment_out_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.change_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_typing_pyi") + + # Mock CONFIG + fake_config = mocker.MagicMock() + fake_config.stub_path = tmp_path / "stubs" + fake_config.repo_path = tmp_path / "repos" + fake_config.typeshed_path = Path("typeshed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.CONFIG", fake_config) + + # Create necessary directories + reference_path = tmp_path / "reference" + reference_path.mkdir(parents=True) + + docstubs_path = tmp_path / "stubs" / "micropython-v1_24_0-docstubs" + docstubs_path.mkdir(parents=True) + + stdlib_path = tmp_path / "publish" / "micropython-stdlib-stubs" / "stdlib" + stdlib_path.mkdir(parents=True, exist_ok=True) + + pyproject = tmp_path / "publish" / "micropython-stdlib-stubs" / "pyproject.toml" + pyproject.parent.mkdir(parents=True, exist_ok=True) + pyproject.write_text("[tool.poetry]\nname='test'\n") + + # Run with merge + result = runner.invoke( + stubber.stubber_cli, + ["stdlib", "--no-build", "--merge", "--no-update"], + ) + + # Should succeed + assert result.exit_code == 0 + + # Verify merge was called + assert m_merge.call_count == 1 + + +@pytest.mark.mocked +def test_cmd_stdlib_stubs_with_update(mocker: MockerFixture, tmp_path: Path): + """Test stdlib command with update option.""" + runner = CliRunner() + + # Mock version + mocker.patch( + "stubber.commands.stdlib_stubs_cmd.get_stable_mp_version", + return_value="1.24.0", + ) + + # Mock update function + m_update_typeshed = mocker.patch("stubber.commands.stdlib_stubs_cmd.update_stdlib_from_typeshed") + + # Mock other functions + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_mpy_shed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_asyncio_manual") + mocker.patch("stubber.commands.stdlib_stubs_cmd.do_post_processing") + mocker.patch("stubber.commands.stdlib_stubs_cmd.add_type_ignore") + mocker.patch("stubber.commands.stdlib_stubs_cmd.comment_out_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.change_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_typing_pyi") + + # Mock CONFIG + fake_config = mocker.MagicMock() + fake_config.stub_path = tmp_path / "stubs" + fake_config.repo_path = tmp_path / "repos" + fake_config.typeshed_path = Path("typeshed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.CONFIG", fake_config) + + # Create necessary directories + reference_path = tmp_path / "reference" + reference_path.mkdir(parents=True) + + typeshed_path = tmp_path / "repos" / "typeshed" + typeshed_path.mkdir(parents=True) + + stdlib_path = tmp_path / "publish" / "micropython-stdlib-stubs" / "stdlib" + stdlib_path.mkdir(parents=True, exist_ok=True) + + pyproject = tmp_path / "publish" / "micropython-stdlib-stubs" / "pyproject.toml" + pyproject.parent.mkdir(parents=True, exist_ok=True) + pyproject.write_text("[tool.poetry]\nname='test'\n") + + # Run with update option + result = runner.invoke( + stubber.stubber_cli, + ["stdlib", "--no-build", "--no-merge", "--update"], + ) + + # Should succeed + assert result.exit_code == 0 + + # Verify update was called + assert m_update_typeshed.call_count == 1 + + +@pytest.mark.mocked +def test_cmd_stdlib_stubs_version_option(mocker: MockerFixture, tmp_path: Path): + """Test stdlib command with explicit version.""" + runner = CliRunner() + + # Don't mock get_stable_mp_version since we're providing version explicitly + m_version = mocker.patch( + "stubber.commands.stdlib_stubs_cmd.get_stable_mp_version", + return_value="1.24.0", + ) + + # Mock other functions + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_mpy_shed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_asyncio_manual") + mocker.patch("stubber.commands.stdlib_stubs_cmd.do_post_processing") + mocker.patch("stubber.commands.stdlib_stubs_cmd.add_type_ignore") + mocker.patch("stubber.commands.stdlib_stubs_cmd.comment_out_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.change_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_typing_pyi") + + # Mock CONFIG + fake_config = mocker.MagicMock() + fake_config.stub_path = tmp_path / "stubs" + fake_config.repo_path = tmp_path / "repos" + fake_config.typeshed_path = Path("typeshed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.CONFIG", fake_config) + + # Create necessary directories + reference_path = tmp_path / "reference" + reference_path.mkdir(parents=True) + + stdlib_path = tmp_path / "publish" / "micropython-stdlib-stubs" / "stdlib" + stdlib_path.mkdir(parents=True, exist_ok=True) + + pyproject = tmp_path / "publish" / "micropython-stdlib-stubs" / "pyproject.toml" + pyproject.parent.mkdir(parents=True, exist_ok=True) + pyproject.write_text("[tool.poetry]\nname='test'\n") + + # Run with explicit version + result = runner.invoke( + stubber.stubber_cli, + ["stdlib", "--no-build", "--no-merge", "--no-update", "-v", "1.23.0"], + ) + + # Should succeed without calling get_stable_mp_version + assert result.exit_code == 0 + assert m_version.call_count == 0 + + +########################################################################################## +# Unit tests for helper functions +########################################################################################## + + +def test_extract_error_lines(): + """Test _extract_error_lines function.""" + from stubber.commands.stdlib_stubs_cmd import _extract_error_lines + + # Test with short text + text = "Error line 1\nError line 2\nError line 3" + result = _extract_error_lines(text) + assert result == "Error line 1\nError line 2\nError line 3" + + # Test with long text (more than max_lines) + text = "\n".join([f"Line {i}" for i in range(20)]) + result = _extract_error_lines(text, max_lines=5) + assert result == "Line 15\nLine 16\nLine 17\nLine 18\nLine 19" + + # Test with empty lines + text = "Line 1\n\n\nLine 2\n\nLine 3" + result = _extract_error_lines(text) + assert result == "Line 1\nLine 2\nLine 3" + + +def test_update_stdlib_from_typeshed(tmp_path: Path): + """Test update_stdlib_from_typeshed function.""" + from stubber.commands.stdlib_stubs_cmd import update_stdlib_from_typeshed + + # Create mock typeshed structure + typeshed_path = tmp_path / "typeshed" + typeshed_stdlib = typeshed_path / "stdlib" + typeshed_stdlib.mkdir(parents=True) + + # Create a module file + (typeshed_stdlib / "sys.pyi").write_text("# sys module") + + # Create a package directory + asyncio_dir = typeshed_stdlib / "asyncio" + asyncio_dir.mkdir() + (asyncio_dir / "__init__.pyi").write_text("# asyncio package") + + # Create dist path + dist_stdlib_path = tmp_path / "dist" + dist_stdlib_path.mkdir() + + # Run the function + update_stdlib_from_typeshed(dist_stdlib_path, typeshed_path) + + # Verify files were copied + stdlib_path = dist_stdlib_path / "stdlib" + assert stdlib_path.exists() + assert (stdlib_path / "sys.pyi").exists() + assert (stdlib_path / "asyncio" / "__init__.pyi").exists() + + +def test_update_stdlib_from_typeshed_missing_path(tmp_path: Path): + """Test update_stdlib_from_typeshed with missing typeshed path.""" + from stubber.commands.stdlib_stubs_cmd import update_stdlib_from_typeshed + + typeshed_path = tmp_path / "nonexistent" + dist_stdlib_path = tmp_path / "dist" + dist_stdlib_path.mkdir() + + with pytest.raises(FileNotFoundError): + update_stdlib_from_typeshed(dist_stdlib_path, typeshed_path) + + +def test_update_mpy_shed(tmp_path: Path): + """Test update_mpy_shed function.""" + from stubber.commands.stdlib_stubs_cmd import update_mpy_shed + + # Create mock reference structure + reference_path = tmp_path / "reference" + mpy_shed_src = reference_path / "_mpy_shed" + mpy_shed_src.mkdir(parents=True) + (mpy_shed_src / "test.pyi").write_text("# test") + + # Create dist path + dist_stdlib_path = tmp_path / "dist" + dist_stdlib_path.mkdir() + + # Run the function + update_mpy_shed(reference_path, dist_stdlib_path) + + # Verify files were copied + mpy_shed_dst = dist_stdlib_path / "stdlib" / "_mpy_shed" + assert mpy_shed_dst.exists() + assert (mpy_shed_dst / "test.pyi").exists() + + +def test_update_asyncio_manual(tmp_path: Path): + """Test update_asyncio_manual function.""" + from stubber.commands.stdlib_stubs_cmd import update_asyncio_manual + + # Create mock reference structure + reference_path = tmp_path / "reference" + asyncio_src = reference_path / "asyncio" + asyncio_src.mkdir(parents=True) + (asyncio_src / "__init__.pyi").write_text("# asyncio") + + # Create dist path + dist_stdlib_path = tmp_path / "dist" + dist_stdlib_path.mkdir() + + # Run the function + update_asyncio_manual(reference_path, dist_stdlib_path) + + # Verify files were copied + asyncio_dst = dist_stdlib_path / "stdlib" / "asyncio" + assert asyncio_dst.exists() + assert (asyncio_dst / "__init__.pyi").exists() + + +def test_merge_docstubs_into_stdlib(tmp_path: Path, mocker: MockerFixture): + """Test merge_docstubs_into_stdlib function.""" + from stubber.commands.stdlib_stubs_cmd import merge_docstubs_into_stdlib + + # Create mock paths + dist_stdlib_path = tmp_path / "dist" + stdlib_path = dist_stdlib_path / "stdlib" + stdlib_path.mkdir(parents=True) + + docstubs_path = tmp_path / "docstubs" + docstubs_path.mkdir() + + boardstub_path = tmp_path / "boardstubs" + boardstub_path.mkdir() + + # Mock enrich_folder + m_enrich = mocker.patch("stubber.commands.stdlib_stubs_cmd.enrich_folder", return_value=5) + + # Run the function + merge_docstubs_into_stdlib(dist_stdlib_path, docstubs_path, boardstub_path) + + # Verify enrich_folder was called twice + assert m_enrich.call_count == 2 + + +def test_add_type_ignore(tmp_path: Path): + """Test add_type_ignore function.""" + from stubber.commands.stdlib_stubs_cmd import add_type_ignore + + # Create a test file + stdlib_path = tmp_path / "stdlib" + stdlib_path.mkdir() + test_file = stdlib_path / "os.pyi" + test_file.write_text("path = _path\ndef other(): pass") + + # Run the function + add_type_ignore(stdlib_path) + + # Verify type ignore was added + content = test_file.read_text() + assert "# type: ignore" in content + assert "path = _path # type: ignore" in content + + +def test_comment_out_lines(tmp_path: Path): + """Test comment_out_lines function.""" + from stubber.commands.stdlib_stubs_cmd import comment_out_lines + + # Create a test file + stdlib_path = tmp_path / "stdlib" + stdlib_path.mkdir() + test_file = stdlib_path / "asyncio.pyi" + test_file.write_text("from .subprocess import *\nother_line") + + # Run the function + comment_out_lines(stdlib_path) + + # Verify line was commented out + content = test_file.read_text() + assert "# from .subprocess import *" in content + + +def test_change_lines(tmp_path: Path): + """Test change_lines function.""" + from stubber.commands.stdlib_stubs_cmd import change_lines + + # Create a test file + stdlib_path = tmp_path / "stdlib" + stdlib_path.mkdir() + test_file = stdlib_path / "ssl.pyi" + test_file.write_text("def create_default_context(): pass\nother_line") + + # Run the function + change_lines(stdlib_path) + + # Verify line was changed + content = test_file.read_text() + assert "def __mpy_has_no_create_default_context" in content + + +def test_update_typing_pyi(tmp_path: Path): + """Test update_typing_pyi function.""" + from stubber.commands.stdlib_stubs_cmd import update_typing_pyi + + # Create paths + rootpath = tmp_path / "root" + dist_stdlib_path = tmp_path / "dist" + + # Run the function (currently does nothing, but should not error) + update_typing_pyi(rootpath, dist_stdlib_path) + + +@pytest.mark.mocked +def test_cmd_stdlib_build_error(mocker: MockerFixture, tmp_path: Path): + """Test stdlib command with build error.""" + runner = CliRunner() + + # Mock version + mocker.patch( + "stubber.commands.stdlib_stubs_cmd.get_stable_mp_version", + return_value="1.24.0", + ) + + # Mock other functions + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_mpy_shed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_asyncio_manual") + mocker.patch("stubber.commands.stdlib_stubs_cmd.do_post_processing") + mocker.patch("stubber.commands.stdlib_stubs_cmd.add_type_ignore") + mocker.patch("stubber.commands.stdlib_stubs_cmd.comment_out_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.change_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_typing_pyi") + + # Mock subprocess to raise error + import subprocess + + mocker.patch( + "stubber.commands.stdlib_stubs_cmd.subprocess.check_call", + side_effect=subprocess.CalledProcessError(1, "uv build", stderr="Build failed"), + ) + + # Mock CONFIG + fake_config = mocker.MagicMock() + fake_config.stub_path = tmp_path / "stubs" + fake_config.repo_path = tmp_path / "repos" + fake_config.typeshed_path = Path("typeshed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.CONFIG", fake_config) + + # Create necessary directories + reference_path = tmp_path / "reference" + reference_path.mkdir(parents=True) + + stdlib_path = tmp_path / "publish" / "micropython-stdlib-stubs" / "stdlib" + stdlib_path.mkdir(parents=True, exist_ok=True) + + pyproject = tmp_path / "publish" / "micropython-stdlib-stubs" / "pyproject.toml" + pyproject.parent.mkdir(parents=True, exist_ok=True) + pyproject.write_text("[tool.poetry]\nname='test'\n") + + # Run with build + result = runner.invoke( + stubber.stubber_cli, + ["stdlib", "--build", "--no-merge", "--no-update"], + ) + + # Should fail with error message + assert result.exit_code != 0 + assert "Build failed" in result.output or "Error" in result.output + + +@pytest.mark.mocked +def test_cmd_stdlib_missing_pyproject(mocker: MockerFixture, tmp_path: Path): + """Test stdlib command with missing pyproject.toml.""" + runner = CliRunner() + + # Mock version + mocker.patch( + "stubber.commands.stdlib_stubs_cmd.get_stable_mp_version", + return_value="1.24.0", + ) + + # Mock other functions + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_mpy_shed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_asyncio_manual") + mocker.patch("stubber.commands.stdlib_stubs_cmd.do_post_processing") + mocker.patch("stubber.commands.stdlib_stubs_cmd.add_type_ignore") + mocker.patch("stubber.commands.stdlib_stubs_cmd.comment_out_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.change_lines") + mocker.patch("stubber.commands.stdlib_stubs_cmd.update_typing_pyi") + + # Mock CONFIG + fake_config = mocker.MagicMock() + fake_config.stub_path = tmp_path / "stubs" + fake_config.repo_path = tmp_path / "repos" + fake_config.typeshed_path = Path("typeshed") + mocker.patch("stubber.commands.stdlib_stubs_cmd.CONFIG", fake_config) + + # Create necessary directories but NOT pyproject.toml + reference_path = tmp_path / "reference" + reference_path.mkdir(parents=True) + + stdlib_path = tmp_path / "publish" / "micropython-stdlib-stubs" / "stdlib" + stdlib_path.mkdir(parents=True, exist_ok=True) + + # Run with build + result = runner.invoke( + stubber.stubber_cli, + ["stdlib", "--build", "--no-merge", "--no-update"], + ) + + # Should fail with error about missing pyproject.toml + assert result.exit_code != 0 + assert "pyproject.toml" in result.output.lower() or "error" in result.output.lower()