Renames

Author: Denis-Averin

scripts/check-urls.py

@@ -1,16 +1,14 @@
 import contextlib
 import fileinput
-import signal
 import os
 import re
 import sys
-import threading
 import typing
 import urllib.parse
 
 from github_job_summary import JobSummary
 from subdomains import Subdomains
-from curl_wrapper import EXIT_CODES as CURL_EXIT_CODES
+from curl_wrapper import CurlExitCodes
 from url_checker import UrlChecker
 
 """
@@ -22,14 +20,14 @@
 
 JOIN_TIMEOUT_SEC = 120
 
-CURL_EXIT_CODES_AND_HTTP_CODES: dict[str, tuple[int, int | None]] = {
-    "https://api.aspose.cloud/connect/token": (CURL_EXIT_CODES.HTTP_RETURNED_ERROR, 400),
-    "https://api.aspose.cloud/v3.0": (CURL_EXIT_CODES.HTTP_RETURNED_ERROR, 404),
-    "https://api.aspose.cloud/v4.0": (CURL_EXIT_CODES.HTTP_RETURNED_ERROR, 404),
-    "https://api.aspose.cloud/v4.0/": (CURL_EXIT_CODES.HTTP_RETURNED_ERROR, 404),
-    "https://id.aspose.cloud/connect/token": (CURL_EXIT_CODES.HTTP_RETURNED_ERROR, 400),
+EXIT_CODE_EXPECTATIONS: dict[str, tuple[int, int | None]] = {
+    "https://api.aspose.cloud/connect/token": (CurlExitCodes.HTTP_RETURNED_ERROR, 400),
+    "https://api.aspose.cloud/v3.0": (CurlExitCodes.HTTP_RETURNED_ERROR, 404),
+    "https://api.aspose.cloud/v4.0": (CurlExitCodes.HTTP_RETURNED_ERROR, 404),
+    "https://api.aspose.cloud/v4.0/": (CurlExitCodes.HTTP_RETURNED_ERROR, 404),
+    "https://id.aspose.cloud/connect/token": (CurlExitCodes.HTTP_RETURNED_ERROR, 400),
     # TODO: Temporary fix
-    "https://dashboard.aspose.cloud/applications": (CURL_EXIT_CODES.HTTP_RETURNED_ERROR, 404),
+    "https://dashboard.aspose.cloud/applications": (CurlExitCodes.HTTP_RETURNED_ERROR, 404),
 }
 
 REGEX_TO_IGNORE: list[re.Pattern[str]] = [
@@ -159,7 +157,7 @@ def text_extractor(files: list[str]) -> typing.Generator[tuple[str, str], None,
 
 def main(files: list[str]) -> int:
     url_checker = UrlChecker(
-        expectations=CURL_EXIT_CODES_AND_HTTP_CODES,
+        expectations=EXIT_CODE_EXPECTATIONS,
     )
 
     with url_checker.start() as checker:
@@ -174,8 +172,8 @@ def main(files: list[str]) -> int:
         if res.ok:
             JOB_SUMMARY.add_success(res.url)
         else:
-            files = EXTRACTED_URLS_WITH_FILES.get(res.url, [])
-            JOB_SUMMARY.add_error(f"Broken URL '{res.url}': {res.stderr}Files: {files}")
+            src_files = EXTRACTED_URLS_WITH_FILES.get(res.url, [])
+            JOB_SUMMARY.add_error(f"Broken URL '{res.url}': {res.stderr}Files: {src_files}")
 
     JOB_SUMMARY.finalize("Checked {total} failed **{failed}**\nGood={success}")
     if JOB_SUMMARY.has_errors:

scripts/curl_wrapper.py

@@ -9,7 +9,11 @@
 DEFAULT_USER_AGENT = "Googlebot/2.1 (+http://www.google.com/bot.html)"
 
 
-class EXIT_CODES:
+class CurlExitCodes:
+    """
+    See: https://curl.se/libcurl/c/libcurl-errors.html
+    """
+
     OK = 0
     COULDNT_RESOLVE_HOST = 6
     HTTP_RETURNED_ERROR = 22
@@ -18,7 +22,6 @@ class EXIT_CODES:
 class CurlWrapper:
     """
     Encapsulates a single curl execution with timeouts and helpers.
-    See: https://curl.se/libcurl/c/libcurl-errors.html
     """
 
     CURL_STDERR_HTTP_RE = re.compile(r"^curl: \(22\) The requested URL returned error: (?P<http_code>\d+)")

scripts/url_checker.py

@@ -1,13 +1,13 @@
 import contextlib
 import sys
+import threading
 import time
 from dataclasses import dataclass
 from queue import Queue, Empty
-from typing import Callable, Optional, Iterable
 from types import TracebackType
-import threading
+from typing import Callable, Optional
 
-from curl_wrapper import CurlWrapper, EXIT_CODES
+from curl_wrapper import CurlWrapper, CurlExitCodes
 
 
 @dataclass
@@ -47,17 +47,17 @@ def __init__(
     def add_url(self, url: str) -> None:
         self.queue.put_nowait(url)
 
-    def close(self) -> None:
+    def _close(self) -> None:
         if not self._closed:
             self._closed = True
             self.queue.put_nowait(None)
 
-    def stop(self) -> None:
+    def _stop(self) -> None:
         self.stop_event = True
         with contextlib.suppress(Exception):
             self.queue.put_nowait(None)
 
-    def run(self) -> None:
+    def _run(self) -> None:
         queue_is_empty = False
         while not queue_is_empty or any(self.workers):
             # Graceful stop: cancel running curls
@@ -105,7 +105,7 @@ def run(self) -> None:
     def start(self) -> "UrlChecker":
         if self._thread is not None:
             return self
-        self._thread = threading.Thread(target=self.run, daemon=True)
+        self._thread = threading.Thread(target=self._run, daemon=True)
         self._thread.start()
         return self
 
@@ -119,20 +119,20 @@ def __exit__(
         tb: TracebackType | None,
     ) -> None:
         # Ensure we signal end of input and wait for completion
-        self.close()
+        self._close()
         self.wait()
 
     def wait(self, join_timeout_sec: float | None = None) -> None:
         # Ensure end-of-input signaled before waiting
-        self.close()
+        self._close()
         t = self._thread
         if t is None:
             return
         if join_timeout_sec is not None:
             t.join(timeout=join_timeout_sec)
             if t.is_alive():
                 # Try to stop gracefully and inform user
-                self.stop()
+                self._stop()
                 print(
                     f"URL checker did not finish within {join_timeout_sec}s; exiting early.",
                     file=sys.stderr,
@@ -155,7 +155,7 @@ def _process_finished(self, task: CurlWrapper) -> None:
             stderr_out = ""
         else:
             # If curl reports HTTP error (22), attempt to parse HTTP code to compare
-            if task.ret_code == EXIT_CODES.HTTP_RETURNED_ERROR and expected_http_code:
+            if task.ret_code == CurlExitCodes.HTTP_RETURNED_ERROR and expected_http_code:
                 match = CurlWrapper.CURL_STDERR_HTTP_RE.match(task.stderr)
                 assert match, "Unexpected output: %s" % task.stderr
                 http_code_val = int(match.groupdict()["http_code"])