Merge pull request #115 from lsst/tickets/DM-38499

DM-38499: Clean up package and change to run flake8/ruff

Author: timj

.github/workflows/build.yaml

@@ -11,36 +11,38 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
+        pyversion: ["3.10", "3.11"]
 
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
 
       - uses: conda-incubator/setup-miniconda@v2
         with:
-          python-version: 3.8
+          python-version: ${{ matrix.pyversion }}
+          auto-update-conda: true
           channels: conda-forge,defaults
+          miniforge-variant: Mambaforge
+          use-mamba: true
           channel-priority: strict
           show-channel-urls: true
 
       - name: Install conda packages used by this package
         shell: bash -l {0}
         run: |
-          conda install mamba
-          mamba install -y -q scons eups compilers
+          mamba install -y -q scons eups compilers ruff flake8
 
       # We have two cores so we can speed up the testing with xdist
       - name: Install pytest packages
         shell: bash -l {0}
         run: |
           mamba install -y -q \
-            "flake8<5" \
-            pytest pytest-flake8 pytest-xdist pytest-openfiles pytest-cov pytest-session2file
+            pytest pytest-xdist pytest-openfiles pytest-cov pytest-session2file
 
       - name: List installed packages
         shell: bash -l {0}
         run: |
-          conda list
+          mamba list
           pip list -v
 
       - name: Build and test

.github/workflows/formatting.yaml

@@ -0,0 +1,11 @@
+name: Check Python formatting
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  call-workflow:
+    uses: lsst/rubin_workflows/.github/workflows/formatting.yaml@main

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-toml
+      - id: check-yaml
+        args:
+          - "--unsafe"
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+      - id: black
+        # It is recommended to specify the latest version of Python
+        # supported by your project here, or alternatively use
+        # pre-commit's default_language_version, see
+        # https://pre-commit.com/#top_level-default_language_version
+        language_version: python3.11
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.0.275
+    hooks:
+      - id: ruff

SConstruct

@@ -3,7 +3,8 @@
 # Setup our environment
 #
 import sys
-sys.path = ["python"] + sys.path # ensure that we use our copy of sconsUtils
+
+sys.path = ["python"] + sys.path  # ensure that we use our copy of sconsUtils
 from lsst.sconsUtils import scripts
 
 scripts.BasicSConstruct.initialize(packageName="sconsUtils")

bin.src/SConscript

@@ -1,3 +1,4 @@
 # -*- python -*-
 from lsst.sconsUtils import scripts
+
 scripts.BasicSConscript.shebang()

bin.src/makeLinkFarm

@@ -1,15 +1,16 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
+
 import argparse
 import os
 import sys
+
 import eups
 
 
 def linkFiles(dstDir, srcDir, verbose=0, force=False):
-    """Link all the regular files in srcDir into dstDir; mkdir all the directories in srcDir and
-    descend recursively"""
+    """Link all the regular files in srcDir into dstDir; mkdir all the
+    directories in srcDir and descend recursively"""
 
     if not os.path.exists(dstDir):
         os.makedirs(dstDir)
@@ -22,40 +23,42 @@ def linkFiles(dstDir, srcDir, verbose=0, force=False):
             linkFiles(fullDst, fullSrc, verbose, force)
         else:
             if verbose > 1:
-                print("%s -> %s" % (fullSrc, fullDst))
+                print(f"{fullSrc} -> {fullDst}")
 
             if os.path.exists(fullDst):
                 if force:
                     os.unlink(fullDst)
                 else:
-                    print("%s already exists; use --force to overwrite" % fullDst, file=sys.stderr)
+                    print(f"{fullDst} already exists; use --force to overwrite", file=sys.stderr)
                     sys.exit(1)
 
             os.symlink(fullSrc, fullDst)
 
 
 def main():
-    parser = argparse.ArgumentParser(description='''Build a link farm to replace long -I/-L argument lists.
+    parser = argparse.ArgumentParser(
+        description="""Build a link farm to replace long -I/-L argument lists.
 All the bin, include files, and libraries for all the dependencies of the specified product are linked into
     linkFarmDir/bin, linkFarmDir/include, or linkFarmDir/lib
 as appropriate.  Note that the setup dependencies are used, so you can say e.g.
 $ setup pipe_tasks
 $ setup -r ~/LSST/afw
 $ makeLinkFarm /tmp/linkFarm pipe_tasks
 creates a link farm for your current selection of products.
-''')
-    parser.add_argument('linkFarmDir', type=str, help='The directory to setup as a link farm')
-    parser.add_argument('productName', type=str, help='The desired product (must be setup)')
+"""
+    )
+    parser.add_argument("linkFarmDir", type=str, help="The directory to setup as a link farm")
+    parser.add_argument("productName", type=str, help="The desired product (must be setup)")
 
-    parser.add_argument('--force', '-f', action='store_true', help="Force creation of farm even if it exists")
-    parser.add_argument('--verbose', '-v', action='count', help="Be chatty")
+    parser.add_argument("--force", "-f", action="store_true", help="Force creation of farm even if it exists")
+    parser.add_argument("--verbose", "-v", action="count", help="Be chatty")
 
     args = parser.parse_args()
 
     linkFarmDir = os.path.abspath(args.linkFarmDir)
 
     if not args.force and os.path.exists(linkFarmDir):
-        print("Path %s already exists; use --force to overwrite" % args.linkFarmDir, file=sys.stderr)
+        print(f"Path {args.linkFarmDir} already exists; use --force to overwrite", file=sys.stderr)
         sys.exit(1)
 
     #
@@ -67,25 +70,25 @@ creates a link farm for your current selection of products.
             os.makedirs(d)
 
     products = {}
-    for productName, versionName, optional, recursionDepth in \
-            eups.getDependencies(args.productName, None, setup=True):
+    for productName, versionName, optional, recursionDepth in eups.getDependencies(
+        args.productName, None, setup=True
+    ):
         if productName in products:
             continue
 
         products[productName] = 1
 
         pd = eups.productDir(productName)
-        if not pd:                      # no product
+        if not pd:  # no product
             continue
 
         binDir = os.path.join(pd, "bin")
         incDir = os.path.join(pd, "include")
         libDir = os.path.join(pd, "lib")
 
         if args.verbose:
-            if args.verbose > 1 or \
-                    os.path.exists(binDir) or os.path.exists(incDir) or os.path.exists(libDir):
-                print("%-15s %s" % (productName, pd))
+            if args.verbose > 1 or os.path.exists(binDir) or os.path.exists(incDir) or os.path.exists(libDir):
+                print(f"{productName:<15} {pd}")
 
         if os.path.exists(binDir):
             fullDstDir = os.path.join(linkFarmDir, "bin")
@@ -108,7 +111,7 @@ creates a link farm for your current selection of products.
                     if args.force:
                         os.unlink(fullDst)
                     else:
-                        print("%s already exists; use --force to overwrite" % fullDst, file=sys.stderr)
+                        print(f"{fullDst} already exists; use --force to overwrite", file=sys.stderr)
                         sys.exit(1)
 
                 os.symlink(fullSrc, fullDst)

bin.src/sconsOpts

@@ -2,23 +2,20 @@
 #
 # Print the options last passed to scons
 #
-from __future__ import print_function
+import argparse
+import configparser
 import os
 import sys
-import argparse
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
 
 parser = argparse.ArgumentParser(description="Tell us how scons was invoked")
-parser.add_argument('configFile', type=str, nargs="?", default=None,
-                    help='The file with the information we need')
-parser.add_argument('--cc', type=str, default="gcc",
-                    help="Use this compiler if build.cfg is unavailable")
-parser.add_argument('--opt', type=int, default=0,
-                    help="Use this optimisation level if build.cfg is unavailable")
-parser.add_argument('--quiet', '-q', action="store_true", help="Don't generate any output")
+parser.add_argument(
+    "configFile", type=str, nargs="?", default=None, help="The file with the information we need"
+)
+parser.add_argument("--cc", type=str, default="gcc", help="Use this compiler if build.cfg is unavailable")
+parser.add_argument(
+    "--opt", type=int, default=0, help="Use this optimisation level if build.cfg is unavailable"
+)
+parser.add_argument("--quiet", "-q", action="store_true", help="Don't generate any output")
 
 args = parser.parse_args()
 dirName = "."
@@ -36,14 +33,14 @@ if os.path.exists(args.configFile):
     try:
         config.read(args.configFile)
 
-        cc = config.get("Build", 'cc')
-        opt = config.get("Build", 'opt')
+        cc = config.get("Build", "cc")
+        opt = config.get("Build", "opt")
     except Exception as e:
         if not args.quiet:
-            print("File %s: error %s" % (args.configFile, e), file=sys.stderr)
+            print(f"File {args.configFile}: error {e}", file=sys.stderr)
 
 else:
     if not args.quiet:
-        print("File %s doesn't exist" % args.configFile, file=sys.stderr)
+        print(f"File {args.configFile} doesn't exist", file=sys.stderr)
 
-print("cc=%s opt=%s" % (cc, opt))
+print(f"cc={cc} opt={opt}")

configs/boost_system.cfg

@@ -12,4 +12,3 @@ config = lsst.sconsUtils.ExternalConfiguration(
     libs=["boost_system"],
     eupsProduct="boost",
 )
-

doc/conf.py

@@ -25,12 +25,10 @@
 This configuration only affects single-package Sphinx documentation builds.
 """
 
-from documenteer.sphinxconfig.stackconf import build_package_configs
-import lsst.sconsUtils
-import lsst.sconsUtils.version
+from documenteer.conf.pipelinespkg import *  # noqa: F403, import *
 
-
-_g = globals()
-_g.update(build_package_configs(
-    project_name="sconsUtils",
-    version=lsst.sconsUtils.version.__version__))
+project = "sconsUtils"
+html_theme_options["logotext"] = project  # noqa: F405, unknown name
+html_title = project
+html_short_title = project
+doxylink = {}

doc/lsst.sconsUtils/index.rst

@@ -51,5 +51,5 @@ Using ``sconsUtils`` without the ``conda`` Compilers
 
 If you would like to use ``sconsUtils`` without the ``conda`` compilers, then put
 ``SCONSUTILS_AVOID_CONDA_COMPILERS`` in your environment with a non-``None`` value.
-This environment variable will instruct ``sconsUtils`` to use the default system 
+This environment variable will instruct ``sconsUtils`` to use the default system
 compilers and compiler flags.