Query objects.inv from homepage url (#40)

* ✨ query objects.inv from homepage url

* 🎨 pre-commit run

* 💡 fix typehints

* ⚡ cache metadata

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* 🎨 add blank line after docstring

* ✅ fix seeding test assertion

* 🚨 make mypy happy

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* ✅ fix packaging url for test

* ✅ fix numpy error check

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Minor nits

---------

Co-authored-by: mathbou <m.bouzard@nki.local>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Mathieu <923463-mathbou@users.noreply.gitlab.com>
Co-authored-by: Dominic Davis-Foster <dominic@davis-foster.co.uk>

Author: 5 people

seed_intersphinx_mapping/__init__.py

@@ -33,8 +33,10 @@
 # stdlib
 import functools
 import json
+import os.path
 import re
 from typing import Dict, Optional, Tuple, Union
+from urllib.parse import urlparse
 
 # 3rd party
 import dist_meta
@@ -74,24 +76,29 @@ def _get_project_links(project_name: str) -> MetadataMapping:
 	# Try a local package first
 	try:
 		dist = dist_meta.distributions.get_distribution(project_name)
-		raw_urls = dist.get_metadata().get_all("Project-URL", default=())
+		metadata = dist.get_metadata()
+		raw_urls = metadata.get_all("Project-URL", default=())
 
 		for url in raw_urls:
-			label, url, *_ = map(str.strip, url.split(','))
+			label, url = map(str.strip, url.split(',', 1))
 			if _DOCUMENTATION_RE.match(label):
 				urls[label] = url
 
+		urls["Home-Page"] = metadata.get("Home-Page", '')
+
 	except dist_meta.distributions.DistributionNotFoundError:
 		# Fall back to PyPI
 
 		with PyPIJSON() as client:
-			metadata = client.get_metadata(project_name).info
+			pypi_metadata = client.get_metadata(project_name).info
 
-		if "project_urls" in metadata and metadata["project_urls"]:
-			for label, url in metadata["project_urls"].items():
+		if "project_urls" in pypi_metadata and pypi_metadata["project_urls"]:
+			for label, url in pypi_metadata["project_urls"].items():
 				if _DOCUMENTATION_RE.match(label):
 					urls[label] = url
 
+		urls["Homepage"] = pypi_metadata.get("home_page", '')
+
 	return urls
 
 
@@ -127,25 +134,27 @@ def get_sphinx_doc_url(pypi_name: str) -> str:
 		:exc:`apeye.slumber_url.exceptions.HttpNotFoundError` if the project could not be found on PyPI.
 	"""
 
-	for key, value in _get_project_links(pypi_name).items():
+	docs_urls = []
+	for value in _get_project_links(pypi_name).values():
+		if not value:
+			continue
 
 		# Follow redirects to get actual URL
 		r = requests.head(value, allow_redirects=True, timeout=10)
-		if r.status_code != 200:  # pragma: no cover
-			raise ValueError(f"Documentation URL not found: HTTP Status {r.status_code}.")
 
-		docs_url = r.url
+		if r.status_code == 200:
+			has_extension = os.path.splitext(urlparse(r.url).path)[-1]
+			url = os.path.dirname(r.url) if has_extension else r.url
+			docs_urls.append(url)
 
-		if docs_url.endswith('/'):
-			objects_inv_url = f"{docs_url}objects.inv"
-		else:  # pragma: no cover
-			objects_inv_url = f"{docs_url}/objects.inv"
+	for docs_url in docs_urls:
+		objects_inv_url = f"{docs_url.rstrip('/')}/objects.inv"
 
 		r = requests.head(objects_inv_url, allow_redirects=True, timeout=10)
 		if r.status_code != 200:
-			raise ValueError(f"objects.inv not found at url {objects_inv_url}: HTTP Status {r.status_code}.")
-
-		return docs_url
+			stderr_writer(f"WARNING: objects.inv not found at url {objects_inv_url}: HTTP Status {r.status_code}.")
+		else:
+			return docs_url
 
 	raise ValueError("Documentation URL not found in data from PyPI.")
 

seed_intersphinx_mapping/fallback_mapping.json

@@ -30,7 +30,7 @@
   "scikit-learn": "https://scikit-learn.org/stable/",
   "six": "https://six.readthedocs.io/",
   "slumber": "https://slumber.readthedocs.io/en/v0.6.0/",
-  "sphinx": "https://www.sphinx-doc.org/en/3.x/",
+  "sphinx": "https://www.sphinx-doc.org/en/master/",
   "typing": "https://docs.python.org/3/",
   "typing-extensions": "https://typing-extensions.readthedocs.io/en/latest/",
   "typing_extensions": "https://typing-extensions.readthedocs.io/en/latest/",

tests/test_core.py

@@ -24,7 +24,7 @@ def test_get_sphinx_doc_url():
 	with pytest.raises(ValueError, match="Documentation URL not found in data from PyPI."):
 		get_sphinx_doc_url("slumber")
 
-	with pytest.raises(ValueError, match="objects.inv not found at url .*: HTTP Status 404"):
+	with pytest.raises(ValueError, match="Documentation URL not found in data from PyPI."):
 		get_sphinx_doc_url("autodoc_pydantic")
 
 	assert cache.clear(get_sphinx_doc_url)
@@ -35,10 +35,7 @@ def test_get_sphinx_doc_url():
 
 	if sys.version_info[:2] != (3, 8):
 		# Latest numpy's "Documentation" url doesn't point to Sphinx docs.
-		with pytest.raises(
-				ValueError,
-				match="objects.inv not found at url https://numpy.org/doc/objects.inv: HTTP Status 404."
-				):
+		with pytest.raises(ValueError, match="Documentation URL not found in data from PyPI."):
 			get_sphinx_doc_url("numpy")
 	else:
 		assert re.match(r"https://numpy\.org/doc/1\.\d\d/", get_sphinx_doc_url("numpy"))

tests/test_seeding.py

@@ -25,12 +25,12 @@
 		"packaging": ("https://packaging.pypa.io/en/stable/", None),
 		"requests": ("https://requests.readthedocs.io/en/latest/", None),
 		"slumber": ("https://slumber.readthedocs.io/en/v0.6.0/", None),
-		"sphinx": ("https://www.sphinx-doc.org/en/3.x/", None),
+		"sphinx": ("https://www.sphinx-doc.org/en/master/", None),
 		}
 bad_expected_mapping = {
 		"domdf-python-tools": ("https://domdf-python-tools.readthedocs.io/en/latest/", None),
 		"packaging": ("https://packaging.pypa.io/en/stable/", None),
-		"sphinx": ("https://www.sphinx-doc.org/en/3.x/", None),
+		"sphinx": ("https://www.sphinx-doc.org/en/master/", None),
 		}
 
 
@@ -50,12 +50,12 @@ def test_seed_intersphinx_mapping(
 
 	assert seed_intersphinx_mapping(*parse_requirements_txt(tmp_pathplus)) == expects
 	err = capsys.readouterr().err
-	assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n"
+	assert "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n" in err
 
 	requirements, comments, invalid = read_requirements(tmp_pathplus / "requirements.txt", include_invalid=True)
 	assert seed_intersphinx_mapping(*requirements) == expects
 	err = capsys.readouterr().err
-	assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n"
+	assert "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n" in err
 
 
 @pytest.mark.parametrize(
@@ -70,7 +70,7 @@ def test_seed_intersphinx_mapping_pyproject(tmp_pathplus: PathPlus, contents: st
 
 	assert seed_intersphinx_mapping(*parse_pyproject_toml(tmp_pathplus)) == expects
 	err = capsys.readouterr().err
-	assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n"
+	assert err.endswith("WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n")
 
 
 @pytest.mark.parametrize(
@@ -85,7 +85,7 @@ def test_seed_intersphinx_mapping_flit(tmp_pathplus: PathPlus, contents: str, ex
 
 	assert seed_intersphinx_mapping(*parse_flit_requirements(tmp_pathplus)) == expects
 	err = capsys.readouterr().err
-	assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n"
+	assert err.endswith("WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n")
 
 
 @pytest.mark.parametrize("pkg_requirements_source", ["requirements", "flit", "pyproject", "pyproject.toml"])
@@ -125,7 +125,7 @@ def test_sphinx_seed_intersphinx_mapping_mocked(
 	advanced_data_regression.check(config.intersphinx_mapping)
 
 	err = capsys.readouterr().err
-	assert err == "WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n"
+	assert err.endswith("WARNING: Unable to determine documentation url for project sphinxcontrib-domaintools\n")
 
 
 def test_sphinx_seed_intersphinx_mapping_list_mocked(

tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_list_mocked.yml

@@ -16,5 +16,5 @@ slumber:
   - - null
 sphinx:
 - sphinx
-- - https://www.sphinx-doc.org/en/3.x/
+- - https://www.sphinx-doc.org/en/master/
   - - null

tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_flit_.yml

@@ -8,5 +8,5 @@ packaging:
   - - null
 sphinx:
 - sphinx
-- - https://www.sphinx-doc.org/en/3.x/
+- - https://www.sphinx-doc.org/en/master/
   - - null

tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_.yml

@@ -8,5 +8,5 @@ packaging:
   - - null
 sphinx:
 - sphinx
-- - https://www.sphinx-doc.org/en/3.x/
+- - https://www.sphinx-doc.org/en/master/
   - - null

tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_pyproject_toml_.yml

@@ -8,5 +8,5 @@ packaging:
   - - null
 sphinx:
 - sphinx
-- - https://www.sphinx-doc.org/en/3.x/
+- - https://www.sphinx-doc.org/en/master/
   - - null

tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_bad_example_requirements_requirements_.yml

@@ -8,5 +8,5 @@ packaging:
   - - null
 sphinx:
 - sphinx
-- - https://www.sphinx-doc.org/en/3.x/
+- - https://www.sphinx-doc.org/en/master/
   - - null

tests/test_seeding_/test_sphinx_seed_intersphinx_mapping_mocked_example_requirements_a_flit_.yml

@@ -16,5 +16,5 @@ slumber:
   - - null
 sphinx:
 - sphinx
-- - https://www.sphinx-doc.org/en/3.x/
+- - https://www.sphinx-doc.org/en/master/
   - - null