fix flake8 check

Author: mr-c

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+ignore = E203,W503
+max-line-length = 100
+select = B,C,E,F,W,T4
+exclude = cwltool/schemas
+extend-ignore = E501,B905
+# when Python 3.10 is the minimum version, re-enable check B905 for zip + strict
+extend-select = B9

Makefile

@@ -109,8 +109,8 @@ codespell:
 	codespell -w $(shell git ls-files | grep -v mypy-stubs)
 
 ## format                 : check/fix all code indentation and formatting (runs black)
-format: $(PYSOURCES)
-	black $^
+format: $(PYSOURCES) FORCE
+	black $(PYSOURCES)
 
 format-check: $(PYSOURCES)
 	black --diff --check $^

cwl_utils/cwl_v1_0_expression_refactor.py

@@ -350,7 +350,7 @@ def generate_etool_from_expr(
         elif self_type.type:
             new_type = clean_type_ids(self_type.type)
         else:
-            raise WorkflowException(f"Don't know how to make type from '{self_type}'.")
+            raise WorkflowException(f"Don't know how to make type from {self_type!r}.")
         inputs.append(
             cwl.InputParameter(
                 id="self",
@@ -652,7 +652,8 @@ def process_workflow_reqs_and_hints(
     """
     # TODO: consolidate the generated etools/cltools into a single "_expression_workflow_reqs" step
     # TODO: support resourceReq.* references to Workflow.inputs?
-    #       ^ By refactoring replace_expr_etool to allow multiple inputs, and connecting all workflow inputs to the generated step
+    #       ^ By refactoring replace_expr_etool to allow multiple inputs,
+    #         and connecting all workflow inputs to the generated step
     modified = False
     inputs = empty_inputs(workflow)
     generated_res_reqs: List[Tuple[str, Union[int, str]]] = []
@@ -837,7 +838,8 @@ def process_workflow_reqs_and_hints(
                                                     raise_type=WorkflowException,
                                                 ).makeError(
                                                     f"Entry {index},"
-                                                    + "Invalid CWL, if 'entry' is a string, then entryName must be specified."
+                                                    + "Invalid CWL, if 'entry' "
+                                                    "is a string, then entryName must be specified."
                                                 )
                                             expr = (
                                                 '${return {"class": "File", "basename": "'
@@ -979,7 +981,8 @@ def process_level_reqs(
     # - Replace the CWL Expression inplace with a CWL parameter reference
     # - Don't create a new Requirement, nor delete the existing Requirement
     # - the Process is passed to replace_expr_with_etool for later searching for JS expressionLibs
-    # - in addition to adding the input to the step for the ETool/CTool result, add it to the Process.inputs as well
+    # - in addition to adding the input to the step for the ETool/CTool result,
+    #   add it to the Process.inputs as well
     if not process.requirements:
         return False
     modified = False
@@ -1122,7 +1125,7 @@ def process_level_reqs(
                                         entryname = (
                                             entry.entryname
                                             if entryname_expr
-                                            else f'"{entry.entryname}"'
+                                            else f'"{entry.entryname}"'  # noqa: B907
                                         )
                                         new_expression = (
                                             "${var result; var entryname = "
@@ -1914,15 +1917,15 @@ def workflow_step_to_InputParameters(
                 for p in param:
                     if not p.type:
                         raise WorkflowException(
-                            f"Don't know how to get type id for '{p}'."
+                            f"Don't know how to get type id for {p!r}."
                         )
                     p.id = inp_id
                     p.type = clean_type_ids(p.type)
                     params.append(p)
             else:
                 if not param.type:
                     raise WorkflowException(
-                        f"Don't know how to get type id for '{param}'."
+                        f"Don't know how to get type id for {param!r}."
                     )
                 param.id = inp_id
                 param.type = clean_type_ids(param.type)

cwl_utils/cwl_v1_1_expression_refactor.py

@@ -183,7 +183,7 @@ def etool_to_cltool(
         contents += "\n" + "\n".join(expressionLib)
     contents += (
         """
-    var ret = function(){"""
+var ret = function(){"""
         + escape_expression_field(etool.expression.strip()[2:-1])
         + """}();
 process.stdout.write(JSON.stringify(ret));"""
@@ -389,7 +389,7 @@ def generate_etool_from_expr(
         expression += "\n  var self=inputs.self;"
     expression += (
         """
-      return {"result": function(){"""
+  return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""
@@ -652,7 +652,8 @@ def process_workflow_reqs_and_hints(
     """
     # TODO: consolidate the generated etools/cltools into a single "_expression_workflow_reqs" step
     # TODO: support resourceReq.* references to Workflow.inputs?
-    #       ^ By refactoring replace_expr_etool to allow multiple inputs, and connecting all workflow inputs to the generated step
+    #       ^ By refactoring replace_expr_etool to allow multiple inputs,
+    #         and connecting all workflow inputs to the generated step
     modified = False
     inputs = empty_inputs(workflow)
     generated_res_reqs: List[Tuple[str, Union[int, str]]] = []
@@ -837,7 +838,8 @@ def process_workflow_reqs_and_hints(
                                                     raise_type=WorkflowException,
                                                 ).makeError(
                                                     f"Entry {index},"
-                                                    + "Invalid CWL, if 'entry' is a string, then entryName must be specified."
+                                                    + "Invalid CWL, if 'entry' "
+                                                    "is a string, then entryName must be specified."
                                                 )
                                             expr = (
                                                 '${return {"class": "File", "basename": "'
@@ -979,7 +981,8 @@ def process_level_reqs(
     # - Replace the CWL Expression inplace with a CWL parameter reference
     # - Don't create a new Requirement, nor delete the existing Requirement
     # - the Process is passed to replace_expr_with_etool for later searching for JS expressionLibs
-    # - in addition to adding the input to the step for the ETool/CTool result, add it to the Process.inputs as well
+    # - in addition to adding the input to the step for the ETool/CTool result,
+    #   add it to the Process.inputs as well
     if not process.requirements:
         return False
     modified = False
@@ -1122,7 +1125,7 @@ def process_level_reqs(
                                         entryname = (
                                             entry.entryname
                                             if entryname_expr
-                                            else f'"{entry.entryname}"'
+                                            else f'"{entry.entryname}"'  # noqa: B907
                                         )
                                         new_expression = (
                                             "${var result; var entryname = "
@@ -1716,7 +1719,7 @@ def generate_etool_from_expr2(
         expression += f"\n  var self=inputs.{self_name};"
     expression += (
         """
-      return {"result": function(){"""
+  return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""

cwl_utils/cwl_v1_2_expression_refactor.py

@@ -183,7 +183,7 @@ def etool_to_cltool(
         contents += "\n" + "\n".join(expressionLib)
     contents += (
         """
-    var ret = function(){"""
+var ret = function(){"""
         + escape_expression_field(etool.expression.strip()[2:-1])
         + """}();
 process.stdout.write(JSON.stringify(ret));"""
@@ -389,7 +389,7 @@ def generate_etool_from_expr(
         expression += "\n  var self=inputs.self;"
     expression += (
         """
-      return {"result": function(){"""
+  return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""
@@ -747,7 +747,8 @@ def process_workflow_reqs_and_hints(
     """
     # TODO: consolidate the generated etools/cltools into a single "_expression_workflow_reqs" step
     # TODO: support resourceReq.* references to Workflow.inputs?
-    #       ^ By refactoring replace_expr_etool to allow multiple inputs, and connecting all workflow inputs to the generated step
+    #       ^ By refactoring replace_expr_etool to allow multiple inputs,
+    #         and connecting all workflow inputs to the generated step
     modified = False
     inputs = empty_inputs(workflow)
     generated_res_reqs: List[Tuple[str, Union[int, str]]] = []
@@ -932,7 +933,8 @@ def process_workflow_reqs_and_hints(
                                                     raise_type=WorkflowException,
                                                 ).makeError(
                                                     f"Entry {index},"
-                                                    + "Invalid CWL, if 'entry' is a string, then entryName must be specified."
+                                                    + "Invalid CWL, if 'entry' "
+                                                    "is a string, then entryName must be specified."
                                                 )
                                             expr = (
                                                 '${return {"class": "File", "basename": "'
@@ -1074,7 +1076,8 @@ def process_level_reqs(
     # - Replace the CWL Expression inplace with a CWL parameter reference
     # - Don't create a new Requirement, nor delete the existing Requirement
     # - the Process is passed to replace_expr_with_etool for later searching for JS expressionLibs
-    # - in addition to adding the input to the step for the ETool/CTool result, add it to the Process.inputs as well
+    # - in addition to adding the input to the step for the ETool/CTool result,
+    #   add it to the Process.inputs as well
     if not process.requirements:
         return False
     modified = False
@@ -1217,7 +1220,7 @@ def process_level_reqs(
                                         entryname = (
                                             entry.entryname
                                             if entryname_expr
-                                            else f'"{entry.entryname}"'
+                                            else f'"{entry.entryname}"'  # noqa: B907
                                         )
                                         new_expression = (
                                             "${var result; var entryname = "
@@ -1811,7 +1814,7 @@ def generate_etool_from_expr2(
         expression += f"\n  var self=inputs.{self_name};"
     expression += (
         """
-      return {"result": function(){"""
+  return {"result": function(){"""
         + expr[2:-2]
         + """}()};
  }"""

cwl_utils/expression.py

@@ -206,7 +206,7 @@ def dump(string: str) -> str:
     w = scanner(scan)
     while w:
         if convert_to_expression:
-            parts.append(f'"{scan[0: w[0]]}" + ')
+            parts.append(f'"{scan[0: w[0]]}" + ')  # noqa: B907
         else:
             parts.append(scan[0 : w[0]])
 
@@ -227,7 +227,8 @@ def dump(string: str) -> str:
                 parts.append(
                     "function(){var item ="
                     + scan[w[0] : w[1]][2:-1]
-                    + '; if (typeof(item) === "string"){ return item; } else { return JSON.stringify(item); }}() + '
+                    + '; if (typeof(item) === "string"){ return item; } '
+                    "else { return JSON.stringify(item); }}() + "
                 )
         elif scan[w[0]] == "\\":
             if escaping_behavior == 1:
@@ -254,7 +255,7 @@ def dump(string: str) -> str:
         scan = scan[w[1] :]
         w = scanner(scan)
     if convert_to_expression:
-        parts.append(f'"{scan}"')
+        parts.append(f'"{scan}"')  # noqa: B907
         parts.append(";}")
     else:
         parts.append(scan)

cwl_utils/expression_refactor.py

@@ -22,8 +22,14 @@
 from ruamel.yaml.main import YAML
 from ruamel.yaml.scalarstring import walk_tree
 
+from cwl_utils import (
+    cwl_v1_0_expression_refactor,
+    cwl_v1_1_expression_refactor,
+    cwl_v1_2_expression_refactor,
+)
 from cwl_utils.errors import WorkflowException
 from cwl_utils.loghandler import _logger as _cwlutilslogger
+from cwl_utils.parser import cwl_v1_0, cwl_v1_1, cwl_v1_2
 
 if TYPE_CHECKING:
     from typing_extensions import Protocol
@@ -36,13 +42,6 @@
 _logger.setLevel(logging.INFO)
 _cwlutilslogger.setLevel(100)
 
-from cwl_utils import (
-    cwl_v1_0_expression_refactor,
-    cwl_v1_1_expression_refactor,
-    cwl_v1_2_expression_refactor,
-)
-from cwl_utils.parser import cwl_v1_0, cwl_v1_1, cwl_v1_2
-
 save_type = Optional[
     Union[MutableMapping[str, Any], MutableSequence[Any], int, float, bool, str]
 ]
@@ -81,7 +80,8 @@ def arg_parser() -> argparse.ArgumentParser:
     )
     parser.add_argument(
         "--skip-some2",
-        help="Don't process CommandLineTool.outputEval or CommandLineTool.requirements.InitialWorkDirRequirement.",
+        help="Don't process CommandLineTool.outputEval or "
+        "CommandLineTool.requirements.InitialWorkDirRequirement.",
         action="store_true",
     )
     parser.add_argument("dir", help="Directory in which to save converted files")

cwl_utils/image_puller.py

@@ -36,7 +36,7 @@ def _run_command_pull(cmd_pull: List[str]) -> None:
             )
         except subprocess.CalledProcessError as err:
             if err.output:
-                raise subprocess.SubprocessError(err.output)
+                raise subprocess.SubprocessError(err.output) from err
             raise err
 
 
@@ -95,14 +95,12 @@ def get_image_name(self) -> str:
             suffix = ".sif"
         else:
             raise Exception(
-                "Don't know how to handle this version of singularity: {}.".format(
-                    self.version
-                )
+                f"Don't know how to handle this version of singularity: {get_singularity_version()}."
             )
         return f"{image_name}{suffix}"
 
     def save_docker_image(self) -> None:
-        """Pull down the Docker format software container image and save it in the Singularity image format."""
+        """Pull down the Docker container image in the Singularity image format."""
         _LOGGER.info(f"Pulling {self.req} with Singularity...")
         cmd_pull = [
             "singularity",

cwl_utils/normalizer.py

@@ -13,18 +13,17 @@
 from ruamel import yaml
 from schema_salad.sourceline import add_lc_filename
 
+from cwl_utils import cwl_v1_2_expression_refactor
 from cwl_utils.loghandler import _logger as _cwlutilslogger
 from cwl_utils.pack import pack
+from cwl_utils.parser.cwl_v1_2 import load_document_by_yaml, save
 
 _logger = logging.getLogger("cwl-normalizer")  # pylint: disable=invalid-name
 defaultStreamHandler = logging.StreamHandler()  # pylint: disable=invalid-name
 _logger.addHandler(defaultStreamHandler)
 _logger.setLevel(logging.INFO)
 _cwlutilslogger.setLevel(100)
 
-from cwl_utils import cwl_v1_2_expression_refactor
-from cwl_utils.parser.cwl_v1_2 import load_document_by_yaml, save
-
 
 def arg_parser() -> argparse.ArgumentParser:
     """Build the argument parser."""
@@ -44,7 +43,8 @@ def arg_parser() -> argparse.ArgumentParser:
     )
     parser.add_argument(
         "--skip-some2",
-        help="Don't process CommandLineTool.outputEval or CommandLineTool.requirements.InitialWorkDirRequirement.",
+        help="Don't process CommandLineTool.outputEval or "
+        "CommandLineTool.requirements.InitialWorkDirRequirement.",
         action="store_true",
     )
     parser.add_argument(

cwl_utils/pack.py

@@ -13,9 +13,7 @@
 From https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/sbpack/lib.py
 """
 
-
 import logging
-import os
 import sys
 import urllib.parse
 import urllib.request
@@ -216,7 +214,6 @@ def resolve_steps(
     if cwl.get("class") != "Workflow":
         return cwl
 
-    workflow_id = cwl.get("id", os.path.basename(base_url.path))
     for _, v in enumerate(cwl["steps"]):
         if isinstance(v, dict):
             sys.stderr.write(