Merge pull request #121 from lsst/tickets/DM-43369

DM-43369: Drop distutils and modernize ruff usage

Author: timj

.github/workflows/build.yaml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        pyversion: ["3.10", "3.11"]
+        pyversion: ["3.10", "3.11", "3.12"]
 
     runs-on: ${{ matrix.os }}
     steps:

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: check-toml
       - id: check-yaml
@@ -9,7 +9,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 24.3.0
     hooks:
       - id: black
         # It is recommended to specify the latest version of Python
@@ -18,12 +18,12 @@ repos:
         # https://pre-commit.com/#top_level-default_language_version
         language_version: python3.11
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         name: isort (python)
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.0.275
+    rev: v0.3.3
     hooks:
       - id: ruff

pyproject.toml

@@ -7,9 +7,13 @@ profile = "black"
 line_length = 110
 
 [tool.ruff]
+line-length = 110
+target-version = "py311"
 exclude = [
     "__init__.py",
 ]
+
+[tool.ruff.lint]
 ignore = [
     "N802",
     "N803",
@@ -19,22 +23,20 @@ ignore = [
     "N816",
     "N999",
 ]
-line-length = 110
 select = [
     "E",  # pycodestyle
     "F",  # pyflakes
     "N",  # pep8-naming
     "W",  # pycodestyle
 ]
-target-version = "py311"
 extend-select = [
     "RUF100", # Warn about unused noqa
 ]
 
-[tool.ruff.pycodestyle]
+[tool.ruff.lint.pycodestyle]
 max-doc-length = 79
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "numpy"
 
 [tool.pytest.ini_options]

python/lsst/sconsUtils/scripts.py

@@ -5,8 +5,8 @@
 import os.path
 import re
 import shlex
+import shutil
 import warnings
-from distutils.spawn import find_executable
 from stat import ST_MODE
 
 from SCons.Script import BUILD_TARGETS, Dir, File, Glob, SConscript
@@ -247,12 +247,12 @@ def finish(defaultTargets=DEFAULT_TARGETS, subDirList=None, ignoreRegex=None):
             checkTestStatus_command = state.env.Command(
                 "checkTestStatus",
                 [],
-                """
-                @ if [ -d {0} ]; then \
-                      nfail=`find {0} -name "*.failed" | wc -l | sed -e 's/ //g'`; \
+                f"""
+                @ if [ -d {testsDir} ]; then \
+                      nfail=`find {testsDir} -name "*.failed" | wc -l | sed -e 's/ //g'`; \
                       if [ $$nfail -gt 0 ]; then \
                           echo "Failed test output:" >&2; \
-                          for f in `find {0} -name "*.failed"`; do \
+                          for f in `find {testsDir} -name "*.failed"`; do \
                               case "$$f" in \
                               *.xml.failed) \
                                 echo "Global pytest output is in $$f" >&2; \
@@ -263,13 +263,11 @@ def finish(defaultTargets=DEFAULT_TARGETS, subDirList=None, ignoreRegex=None):
                               esac; \
                           done; \
                           echo "The following tests failed:" >&2;\
-                          find {0} -name "*.failed" >&2; \
+                          find {testsDir} -name "*.failed" >&2; \
                           echo "$$nfail tests failed" >&2; exit 1; \
                       fi; \
                   fi; \
-            """.format(
-                    testsDir
-                ),
+            """,
             )
 
             state.env.Depends(checkTestStatus_command, BUILD_TARGETS)  # this is why the check runs last
@@ -372,8 +370,8 @@ def rewrite_shebang(target, source, env):
                         else:
                             if not match:
                                 state.log.warn(
-                                    "Could not rewrite shebang of {}. Please check"
-                                    " file or move it to bin directory.".format(str(src))
+                                    f"Could not rewrite shebang of {src}. Please check"
+                                    " file or move it to bin directory."
                                 )
                             outfd.write(first_line)
                         for line in srcfd.readlines():
@@ -534,7 +532,7 @@ def doc(config="doxygen.conf.in", projectName=None, projectNumber=None, **kwargs
         result : ???
             ???
         """
-        if not find_executable("doxygen"):
+        if not shutil.which("doxygen"):
             state.log.warn("doxygen executable not found; skipping documentation build.")
             return []
         if projectName is None:
@@ -677,8 +675,7 @@ def s(ll):
         for node in pySingles:
             if str(node).startswith("test_"):
                 state.log.warn(
-                    "Warning: {} should be run independently but"
-                    " can be automatically discovered".format(node)
+                    f"Warning: {node} should be run independently but can be automatically discovered"
                 )
 
         # Ensure that python tests listed in pySingles are not included in

python/lsst/sconsUtils/tests.py

@@ -313,6 +313,7 @@ def runPythonLinter(self):
             if no suitable linter configuration was found.
         """
         linter: str | None = None
+        lint_options = ""
         root = SCons.Script.Dir("#").abspath
 
         # Never want linters to look in this directory.
@@ -333,6 +334,7 @@ def runPythonLinter(self):
                 else:
                     if "tool" in parsed and "ruff" in parsed["tool"]:
                         linter = "ruff"
+                        lint_options = "check"
 
                         # Ruff can complain about noqa in shebang line
                         # added by sconsUtils to bin/ scripts. Need to ignore.
@@ -371,8 +373,8 @@ def runPythonLinter(self):
         for path in glob.glob(os.path.join(self._tmpDir, "linter-*.log*")):
             os.unlink(path)
 
-        # Specify exclude directories.
-        lint_options = f"--extend-exclude={','.join(exclude_dirs)}"
+        # Specify exclude directories. Append to any existing lint options.
+        lint_options += f" --extend-exclude={','.join(exclude_dirs)}"
 
         cmd = f"""
         @printf "%s\\n" 'Running python linter {linter}...';