From 765941212edc9d7816fd7b5e1957b834a6b22e33 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Sun, 23 Nov 2025 20:49:16 +0900 Subject: [PATCH] Remove importing compat directly --- micropip/package_manager.py | 11 +++++----- micropip/transaction.py | 8 +++++-- micropip/wheelinfo.py | 41 ++++++++++++++++++++--------------- tests/conftest.py | 23 +++++++++++++++----- tests/test_install.py | 13 +++++------ tests/test_package_manager.py | 27 ++++++++--------------- tests/test_wheelinfo.py | 24 ++++++++++---------- 7 files changed, 80 insertions(+), 67 deletions(-) diff --git a/micropip/package_manager.py b/micropip/package_manager.py index a055424a..602c7fe7 100644 --- a/micropip/package_manager.py +++ b/micropip/package_manager.py @@ -11,7 +11,7 @@ ) from . import _mock_package, package_index -from ._compat import CompatibilityLayer, compatibility_layer +from ._compat import CompatibilityLayer from ._utils import get_files_in_distribution, get_root from ._vendored.packaging.src.packaging.markers import default_environment from .constants import FAQ_URLS @@ -29,10 +29,7 @@ class PackageManager: independent of other instances. """ - def __init__(self, compat: type[CompatibilityLayer] | None = None) -> None: - - if compat is None: - compat = compatibility_layer + def __init__(self, compat: type[CompatibilityLayer]) -> None: self.index_urls = package_index.DEFAULT_INDEX_URLS[:] self.compat_layer: type[CompatibilityLayer] = compat @@ -239,7 +236,9 @@ async def install( # Install PyPI packages # detect whether the wheel metadata is from PyPI or from custom location # wheel metadata from PyPI has SHA256 checksum digest. - await asyncio.gather(*(wheel.install(wheel_base) for wheel in wheels)) + await asyncio.gather( + *(wheel.install(wheel_base, self.compat_layer) for wheel in wheels) + ) # Install built-in packages if pyodide_packages: diff --git a/micropip/transaction.py b/micropip/transaction.py index 702e153b..13aeee82 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -332,13 +332,17 @@ async def add_wheel( logger.info("Collecting %s%s", wheel.name, specifier) logger.info(" Downloading %s", wheel.url.split("/")[-1]) - wheel_download_task = asyncio.create_task(wheel.download(self.fetch_kwargs)) + wheel_download_task = asyncio.create_task( + wheel.download(self.fetch_kwargs, self._compat_layer) + ) if self.deps: # Case 1) If metadata file is available, # we can gather requirements without waiting for the wheel to be downloaded. if wheel.pep658_metadata_available(): try: - await wheel.download_pep658_metadata(self.fetch_kwargs) + await wheel.download_pep658_metadata( + self.fetch_kwargs, self._compat_layer + ) except OSError: # If something goes wrong while downloading the metadata, # we have to wait for the wheel to be downloaded. diff --git a/micropip/wheelinfo.py b/micropip/wheelinfo.py index 2bdc08c6..3ab6b69b 100644 --- a/micropip/wheelinfo.py +++ b/micropip/wheelinfo.py @@ -7,12 +7,7 @@ from typing import Any, Literal from urllib.parse import ParseResult, unquote, urlparse -from ._compat import ( - fetch_bytes, - install, - loadedPackages, - to_js, -) +from ._compat import CompatibilityLayer from ._utils import best_compatible_tag_index, parse_wheel_filename from ._vendored.packaging.src.packaging.requirements import Requirement from ._vendored.packaging.src.packaging.tags import Tag @@ -132,7 +127,9 @@ def from_package_index( _best_tag_index=best_tag_index, ) - async def install(self, target: Path) -> None: + async def install( + self, target: Path, compat_layer: type[CompatibilityLayer] + ) -> None: """ Install the wheel to the target directory. @@ -148,13 +145,15 @@ async def install(self, target: Path) -> None: "Micropip internal error: attempted to install wheel before downloading it?" ) _validate_sha256_checksum(self._data, self.sha256) - await self._install(target) + await self._install(target, compat_layer) - async def download(self, fetch_kwargs: dict[str, Any]): + async def download( + self, fetch_kwargs: dict[str, Any], compat_layer: type[CompatibilityLayer] + ): if self._data is not None: return - self._data = await self._fetch_bytes(self.url, fetch_kwargs) + self._data = await self._fetch_bytes(self.url, fetch_kwargs, compat_layer) # The wheel's metadata might be downloaded separately from the wheel itself. # If it is not downloaded yet or if the metadata is not available, extract it from the wheel. @@ -174,6 +173,7 @@ def pep658_metadata_available(self) -> bool: async def download_pep658_metadata( self, fetch_kwargs: dict[str, Any], + compat_layer: type[CompatibilityLayer], ) -> None: """ Download the wheel's metadata. If the metadata is not available, return None. @@ -181,7 +181,7 @@ async def download_pep658_metadata( if self.core_metadata is None: return None - data = await self._fetch_bytes(self.metadata_url, fetch_kwargs) + data = await self._fetch_bytes(self.metadata_url, fetch_kwargs, compat_layer) match self.core_metadata: case {"sha256": checksum}: # sha256 checksum available @@ -204,14 +204,19 @@ def requires(self, extras: set[str]) -> list[Requirement]: self._requires = requires return requires - async def _fetch_bytes(self, url: str, fetch_kwargs: dict[str, Any]): + async def _fetch_bytes( + self, + url: str, + fetch_kwargs: dict[str, Any], + compat_layer: type[CompatibilityLayer], + ): if self.parsed_url.scheme not in ("https", "http", "emfs", "file"): # Don't raise ValueError it gets swallowed raise TypeError( f"Cannot download from a non-remote location: {url!r} ({self.parsed_url!r})" ) try: - bytes = await fetch_bytes(url, fetch_kwargs) + bytes = await compat_layer.fetch_bytes(url, fetch_kwargs) return bytes except OSError as e: if self.parsed_url.hostname in [ @@ -228,7 +233,9 @@ async def _fetch_bytes(self, url: str, fetch_kwargs: dict[str, Any]): ) from e raise e - async def _install(self, target: Path) -> None: + async def _install( + self, target: Path, compat_layer: type[CompatibilityLayer] + ) -> None: """ Install the wheel to the target directory. """ @@ -247,15 +254,15 @@ async def _install(self, target: Path) -> None: sorted(x.name for x in self._requires) ) - await install( + await compat_layer.install( # TODO: Probably update install API to accept bytes directly, instead of converting it to JS. - to_js(self._data), + compat_layer.to_js(self._data), self.filename, str(target), metadata, ) - setattr(loadedPackages, self._project_name, wheel_source) + setattr(compat_layer.loadedPackages, self._project_name, wheel_source) def _validate_sha256_checksum(data: bytes, expected: str | None = None) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index bc9ff20d..d09338aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -240,7 +240,8 @@ def __eq__(self, other): class mock_fetch_cls: - def __init__(self): + def __init__(self, compat_layer=None): + self._compat_layer = compat_layer self.releases_map = {} self.metadata_map = {} self.top_level_map = {} @@ -351,12 +352,12 @@ def write_file(filename, contents): @pytest.fixture -def mock_fetch(monkeypatch, mock_importlib): - from micropip import package_index, wheelinfo +def mock_fetch(monkeypatch, mock_importlib, host_compat_layer): + from micropip import package_index - result = mock_fetch_cls() + result = mock_fetch_cls(host_compat_layer) monkeypatch.setattr(package_index, "query_package", result.query_package) - monkeypatch.setattr(wheelinfo, "fetch_bytes", result._fetch_bytes) + monkeypatch.setattr(host_compat_layer, "fetch_bytes", result._fetch_bytes) return result @@ -469,3 +470,15 @@ def host_compat_layer(): from micropip._compat._compat_not_in_pyodide import CompatibilityNotInPyodide yield CompatibilityNotInPyodide + + +@pytest.fixture +def host_package_manager(host_compat_layer): + """ + Fixture to provide a package manager for the host environment. + """ + from micropip.package_manager import PackageManager + + package_manager = PackageManager(compat=host_compat_layer) + + yield package_manager diff --git a/tests/test_install.py b/tests/test_install.py index ed2f2190..0eb07e46 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -264,14 +264,13 @@ async def test_install_pre( @pytest.mark.asyncio -async def test_fetch_wheel_fail(monkeypatch, wheel_base): +async def test_fetch_wheel_fail(monkeypatch, wheel_base, host_compat_layer): import micropip - from micropip import wheelinfo def _mock_fetch_bytes(arg, *args, **kwargs): raise OSError(f"Request for {arg} failed with status 404: Not Found") - monkeypatch.setattr(wheelinfo, "fetch_bytes", _mock_fetch_bytes) + monkeypatch.setattr(host_compat_layer, "fetch_bytes", _mock_fetch_bytes) msg = "Access-Control-Allow-Origin" with pytest.raises(ValueError, match=msg): @@ -395,7 +394,9 @@ async def run_test(selenium, url, name, version): @pytest.mark.asyncio -async def test_custom_index_urls(mock_package_index_json_api, monkeypatch): +async def test_custom_index_urls( + mock_package_index_json_api, monkeypatch, host_compat_layer +): mock_server_fake_package = mock_package_index_json_api( pkgs=["fake-pkg-micropip-test"] ) @@ -407,9 +408,7 @@ async def _mock_fetch_bytes(url, *args): _wheel_url = url return b"fake wheel" - from micropip import wheelinfo - - monkeypatch.setattr(wheelinfo, "fetch_bytes", _mock_fetch_bytes) + monkeypatch.setattr(host_compat_layer, "fetch_bytes", _mock_fetch_bytes) try: await micropip.install( diff --git a/tests/test_package_manager.py b/tests/test_package_manager.py index 37c670c3..ca313d6c 100644 --- a/tests/test_package_manager.py +++ b/tests/test_package_manager.py @@ -2,19 +2,10 @@ from conftest import mock_fetch_cls import micropip.package_index as package_index -from micropip.package_manager import PackageManager -def get_test_package_manager() -> PackageManager: - package_manager = PackageManager() - - # TODO: inject necessary constructor parameters - - return package_manager - - -def test_set_index_urls(): - manager = get_test_package_manager() +def test_set_index_urls(host_package_manager): + manager = host_package_manager default_index_urls = package_index.DEFAULT_INDEX_URLS assert manager.index_urls == default_index_urls @@ -34,8 +25,8 @@ def test_set_index_urls(): @pytest.mark.asyncio -async def test_list_packages(mock_fetch: mock_fetch_cls): - manager = get_test_package_manager() +async def test_list_packages(mock_fetch: mock_fetch_cls, host_package_manager): + manager = host_package_manager dummy = "dummy" mock_fetch.add_pkg_version(dummy) @@ -50,8 +41,10 @@ async def test_list_packages(mock_fetch: mock_fetch_cls): @pytest.mark.asyncio -async def test_custom_index_url(mock_package_index_json_api, monkeypatch): - manager = get_test_package_manager() +async def test_custom_index_url( + mock_package_index_json_api, monkeypatch, host_compat_layer, host_package_manager +): + manager = host_package_manager mock_server_fake_package = mock_package_index_json_api( pkgs=["fake-pkg-micropip-test"] @@ -64,9 +57,7 @@ async def _mock_fetch_bytes(url, *args): _wheel_url = url return b"fake wheel" - from micropip import wheelinfo - - monkeypatch.setattr(wheelinfo, "fetch_bytes", _mock_fetch_bytes) + monkeypatch.setattr(host_compat_layer, "fetch_bytes", _mock_fetch_bytes) manager.set_index_urls([mock_server_fake_package]) diff --git a/tests/test_wheelinfo.py b/tests/test_wheelinfo.py index 0a6fa5ef..c933ed47 100644 --- a/tests/test_wheelinfo.py +++ b/tests/test_wheelinfo.py @@ -51,24 +51,24 @@ def test_from_package_index(): @pytest.mark.asyncio -async def test_download(wheel_catalog): +async def test_download(wheel_catalog, host_compat_layer): pytest_wheel = wheel_catalog.get("pytest") wheel = WheelInfo.from_url(pytest_wheel.url) assert wheel._metadata is None - await wheel.download({}) + await wheel.download({}, host_compat_layer) assert wheel._metadata is not None @pytest.mark.asyncio -async def test_requires(wheel_catalog, tmp_path): +async def test_requires(wheel_catalog, tmp_path, host_compat_layer): pytest_wheel = wheel_catalog.get("pytest") wheel = WheelInfo.from_url(pytest_wheel.url) - await wheel.download({}) + await wheel.download({}, host_compat_layer) - wheel._install(tmp_path) + wheel._install(tmp_path, host_compat_layer) requirements_default = [str(r.name) for r in wheel.requires(set())] assert "pluggy" in requirements_default @@ -80,7 +80,7 @@ async def test_requires(wheel_catalog, tmp_path): @pytest.mark.asyncio -async def test_download_pep658_metadata(wheel_catalog): +async def test_download_pep658_metadata(wheel_catalog, host_compat_layer): pytest_wheel = wheel_catalog.get("pytest") sha256 = "dummy-sha256" size = 1234 @@ -98,7 +98,7 @@ async def test_download_pep658_metadata(wheel_catalog): assert wheel_with_metadata.pep658_metadata_available() assert wheel_with_metadata._metadata is None - await wheel_with_metadata.download_pep658_metadata({}) + await wheel_with_metadata.download_pep658_metadata({}, host_compat_layer) assert wheel_with_metadata._metadata is not None # metadata should be calculated from the metadata file @@ -119,7 +119,7 @@ async def test_download_pep658_metadata(wheel_catalog): assert not wheel_without_metadata.pep658_metadata_available() assert wheel_without_metadata._metadata is None - await wheel_without_metadata.download_pep658_metadata({}) + await wheel_without_metadata.download_pep658_metadata({}, host_compat_layer) assert wheel_without_metadata._metadata is None # 3) the metadata extracted from the wheel should be the same @@ -134,14 +134,14 @@ async def test_download_pep658_metadata(wheel_catalog): ) assert wheel._metadata is None - await wheel.download({}) + await wheel.download({}, host_compat_layer) assert wheel._metadata is not None assert wheel._metadata.deps == wheel_with_metadata._metadata.deps @pytest.mark.asyncio -async def test_download_pep658_metadata_checksum(wheel_catalog): +async def test_download_pep658_metadata_checksum(wheel_catalog, host_compat_layer): pytest_wheel = wheel_catalog.get("pytest") sha256 = "dummy-sha256" size = 1234 @@ -158,7 +158,7 @@ async def test_download_pep658_metadata_checksum(wheel_catalog): assert wheel._metadata is None with pytest.raises(RuntimeError, match="Invalid checksum: expected dummy-sha256"): - await wheel.download_pep658_metadata({}) + await wheel.download_pep658_metadata({}, host_compat_layer) checksum = "62eb95408ccec185e7a3b8f354a1df1721cd8f463922f5a900c7bf4b69c5a4e8" # TODO: calculate this from the file wheel = WheelInfo.from_package_index( @@ -172,5 +172,5 @@ async def test_download_pep658_metadata_checksum(wheel_catalog): ) assert wheel._metadata is None - await wheel.download_pep658_metadata({}) + await wheel.download_pep658_metadata({}, host_compat_layer) assert wheel._metadata is not None