Merge branch 'main' into cache-negative-lookups

Author: tetron

.flake8

@@ -1,7 +1,10 @@
 [flake8]
-ignore = E203, E266, E501, W503, E211, E731
-max-line-length = 88
-select = B,C,E,F,W,T4,B9
+ignore = E203,W503
+max-line-length = 100
+select = B,C,E,F,W,T4
+extend-ignore = E501,B905
+# when Python 3.10 is the minimum version, re-enable check B905 for zip + strict
+extend-select = B9
 per-file-ignores =
   schema_salad/metaschema.py:B950
   schema_salad/tests/*.py:B011

Makefile

@@ -220,8 +220,11 @@ release:
 		twine upload testenv2/src/${PACKAGE}/dist/* && \
 		git tag ${VERSION} && git push --tags
 
-flake8: $(PYSOURCES)
-	flake8 $^
+flake8: FORCE
+	flake8 $(PYSOURCES)
+
+schema_salad/metaschema.py: schema_salad/codegen_base.py schema_salad/python_codegen_support.py schema_salad/python_codegen.py schema_salad/metaschema
+	schema-salad-tool --codegen python schema_salad/metaschema/metaschema.yml > $@
 
 FORCE:
 

dev-requirements.txt

@@ -1,7 +1,8 @@
 diff_cover < 7.5
 pylint < 2.16
 pydocstyle < 6.4
-flake8-bugbear < 22.13
+flake8 > 4
+flake8-bugbear < 23.2
 tox < 3.29  # until tox-pyenv is updated for tox 4.x
 tox-pyenv < 1.2
 isort < 5.12

schema_salad/__init__.py

@@ -1,4 +1,4 @@
-"""Salad is a schema language for describing JSON or YAML structured linked data documents."""
+"""A schema language for describing JSON or YAML structured linked data documents."""
 
 import logging
 import sys

schema_salad/avro/schema.py

@@ -605,9 +605,8 @@ def make_avsc_object(json_data: JsonDataType, names: Optional[Names] = None) ->
                 )
             if not (doc is None or isinstance(doc, str) or isinstance(doc, list)):
                 raise SchemaParseException(
-                    '"doc" for type {} must be a string, a list of strings, or None: {}'.format(
-                        atype, json_data
-                    )
+                    f'"doc" for type {atype} must be a string, '
+                    f"a list of strings, or None: {json_data}"
                 )
             if atype == "enum":
                 symbols = json_data.get("symbols")

schema_salad/codegen.py

@@ -43,7 +43,7 @@ def codegen(
     """Generate classes with loaders for the given Schema Salad description."""
     j = schema.extend_and_specialize(i, loader)
 
-    gen = None  # type: Optional[CodeGenBase]
+    gen: Optional[CodeGenBase] = None
     base = schema_metadata.get("$base", schema_metadata.get("id"))
     # ``urlsplit`` decides whether to return an encoded result based
     # on the object type. To ensure the code behaves the same for Py

schema_salad/cpp_codegen.py

@@ -7,23 +7,20 @@
 The generated code requires the libyaml-cpp library & headers
 
 To see an example of usage, look at schema_salad/tests/codegen/cwl.cpp
-which can be combined with the CWL V1.0 schema as shown below:
+which can be combined with the CWL V1.0 schema as shown below::
 
-```
-schema-salad-tool --codegen cpp schema_salad/tests/test_schema/CommonWorkflowLanguage.yml \
-        > cwl_v1_0.h
+  schema-salad-tool --codegen cpp \
+          schema_salad/tests/test_schema/CommonWorkflowLanguage.yml \
+          > cwl_v1_0.h
 
-g++ --std=c++20 -I. -lyaml-cpp schema_salad/tests/codegen/cwl.cpp -o cwl-v1_0-test
-./cwl-v1_0-test
+  g++ --std=c++20 -I. -lyaml-cpp schema_salad/tests/codegen/cwl.cpp -o cwl-v1_0-test
+  ./cwl-v1_0-test
 
-# g++ versions older than version 10 may need "--std=c++2a" instead of "--std=c++20"
-```
+  # g++ versions older than version 10 may need "--std=c++2a" instead of "--std=c++20"
 """
 import re
 from typing import IO, Any, Dict, List, Optional, Tuple, Union, cast
 
-from schema_salad.utils import aslist
-
 from . import _logger
 from .codegen_base import CodeGenBase, TypeDef
 from .exceptions import SchemaException
@@ -161,7 +158,6 @@ def writeDefinition(
         self, target: IO[Any], fullInd: str, ind: str, namespace: str
     ) -> None:
         name = safename(self.name)
-        # target.write(f"{fullInd}std::unique_ptr<{self.typeStr}> {name} = std::make_unique<{self.typeStr}>();\n")
         typeStr = self.typeStr.replace(namespace + "::", "")
         target.write(f"{fullInd}heap_object<{typeStr}> {name};\n")
 
@@ -599,8 +595,6 @@ def parseEnum(self, stype: Dict[str, Any]) -> str:
         return name
 
     def parse(self, items: List[Dict[str, Any]]) -> None:
-        types = {i["name"]: i for i in items}  # type: Dict[str, Any]
-
         for stype in items:
             if "type" in stype and stype["type"] == "documentation":
                 continue

schema_salad/exceptions.py

@@ -16,14 +16,14 @@ def __init__(
     ) -> None:
         super().__init__(msg)
         self.message = self.args[0]
-        self.file = None  # type: Optional[str]
-        self.start = None  # type: Optional[Tuple[int, int]]
-        self.end = None  # type: Optional[Tuple[int, int]]
+        self.file: Optional[str] = None
+        self.start: Optional[Tuple[int, int]] = None
+        self.end: Optional[Tuple[int, int]] = None
 
-        self.is_warning = False  # type: bool
+        self.is_warning: bool = False
 
         # It will be set by its parent
-        self.bullet = ""  # type: str
+        self.bullet: str = ""
 
         def simplify(exc: "SchemaSaladException") -> List["SchemaSaladException"]:
             return [exc] if len(exc.message) else exc.children
@@ -36,7 +36,7 @@ def with_bullet(
             return exc
 
         if children is None:
-            self.children = []  # type: List["SchemaSaladException"]
+            self.children: List["SchemaSaladException"] = []
         elif len(children) <= 1:
             self.children = sum((simplify(c) for c in children), [])
         else:
@@ -82,10 +82,10 @@ def leaves(self) -> List["SchemaSaladException"]:
         return []
 
     def prefix(self) -> str:
-        pre = ""  # type:str
+        pre: str = ""
         if self.file:
-            linecol0 = ""  # type: Union[int, str]
-            linecol1 = ""  # type: Union[int, str]
+            linecol0: Union[int, str] = ""
+            linecol1: Union[int, str] = ""
             if self.start:
                 linecol0, linecol1 = self.start
             pre = f"{self.file}:{linecol0}:{linecol1}: "

schema_salad/makedoc.py

@@ -45,7 +45,7 @@ def vocab_type_name(url: str) -> str:
 
 
 def has_types(items: Any) -> List[str]:
-    r = []  # type: List[str]
+    r: List[str] = []
     if isinstance(items, MutableMapping):
         if items["type"] == "https://w3id.org/cwl/salad#record":
             return [items["name"]]
@@ -106,8 +106,9 @@ def block_code(self, code: str, info: Optional[str] = None) -> str:
         return text + ">" + html.escape(code, quote=self._escape) + "</code></pre>\n"
 
 
-def markdown_list_hook(markdown, text, state):
-    # type: (Markdown[str, Any], str, State) -> Tuple[str, State]
+def markdown_list_hook(
+    markdown: "Markdown[str, Any]", text: str, state: "State"
+) -> Tuple[str, "State"]:
     """Patches problematic Markdown lists for later HTML generation.
 
     When a Markdown list with paragraphs not indented with the list
@@ -286,7 +287,8 @@ def __init__(self) -> None:
         self.toc = ""
         self.start_numbering = True
 
-    def add_entry(self, thisdepth, title):  # type: (int, str) -> str
+    def add_entry(self, thisdepth: int, title: str) -> str:
+        """Add an entry to the table of contents."""
         depth = len(self.numbering)
         if thisdepth < depth:
             self.toc += "</ol>"
@@ -390,12 +392,12 @@ def __init__(
     ) -> None:
         self.typedoc = StringIO()
         self.toc = toc
-        self.subs = {}  # type: Dict[str, str]
-        self.docParent = {}  # type: Dict[str, List[str]]
-        self.docAfter = {}  # type: Dict[str, List[str]]
-        self.rendered = set()  # type: Set[str]
+        self.subs: Dict[str, str] = {}
+        self.docParent: Dict[str, List[str]] = {}
+        self.docAfter: Dict[str, List[str]] = {}
+        self.rendered: Set[str] = set()
         self.redirects = redirects
-        self.title = None  # type: Optional[str]
+        self.title: Optional[str] = None
         self.primitiveType = primitiveType
 
         for t in j:
@@ -418,17 +420,15 @@ def __init__(
         metaschema_loader = get_metaschema()[2]
         alltypes = extend_and_specialize(j, metaschema_loader)
 
-        self.typemap = {}  # type: Dict[str, Dict[str, str]]
-        self.uses = {}  # type: Dict[str, List[Tuple[str, str]]]
-        self.record_refs = {}  # type: Dict[str, List[str]]
+        self.typemap: Dict[str, Dict[str, str]] = {}
+        self.uses: Dict[str, List[Tuple[str, str]]] = {}
+        self.record_refs: Dict[str, List[str]] = {}
         for entry in alltypes:
             self.typemap[entry["name"]] = entry
             try:
                 if entry["type"] == "record":
                     self.record_refs[entry["name"]] = []
-                    fields = entry.get(
-                        "fields", []
-                    )  # type: Union[str, List[Dict[str, str]]]
+                    fields: Union[str, List[Dict[str, str]]] = entry.get("fields", [])
                     if isinstance(fields, str):
                         raise KeyError("record fields must be a list of mappings")
                     for f in fields:  # type: Dict[str, str]
@@ -615,20 +615,20 @@ def extendsfrom(item: Dict[str, Any], ex: List[Dict[str, Any]]) -> None:
             f["doc"] = number_headings(self.toc, f["doc"])
 
         doc = doc + "\n\n" + f["doc"]
-        plugins = [
+        plugins: List["PluginName"] = [
             "strikethrough",
             "footnotes",
             "table",
             "url",
-        ]  # type: List[PluginName]  # fix error Generic str != explicit Literals
+        ]
         # if escape active, wraps literal HTML into '<p> {HTML} </p>'
         # we must pass it to both since 'MyRenderer' is predefined
         escape = False
-        markdown2html = create_markdown(
+        markdown2html: "Markdown[str, Any]" = create_markdown(
             renderer=MyRenderer(escape=escape),
             plugins=plugins,
             escape=escape,
-        )  # type: Markdown[str, Any]
+        )
         markdown2html.before_parse_hooks.append(markdown_list_hook)
         doc = markdown2html(doc)
 

schema_salad/metaschema.py

@@ -429,8 +429,7 @@ def __repr__(self):  # type: () -> str
 
 
 class _EnumLoader(_Loader):
-    def __init__(self, symbols, name):
-        # type: (Sequence[str], str) -> None
+    def __init__(self, symbols: Sequence[str], name: str) -> None:
         self.symbols = symbols
         self.name = name
 
@@ -547,8 +546,7 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None):
 
 
 class _UnionLoader(_Loader):
-    def __init__(self, alternates):
-        # type: (Sequence[_Loader]) -> None
+    def __init__(self, alternates: Sequence[_Loader]) -> None:
         self.alternates = alternates
 
     def load(self, doc, baseuri, loadingOptions, docRoot=None):