Merge branch 'common-workflow-language:main' into main

Author: giuseppe998e

.github/workflows/wheels.yml

@@ -44,7 +44,7 @@ jobs:
       #     platforms: all
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@v2.23.2
         env:
           CIBW_BUILD: ${{ matrix.build }}
           CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }}
@@ -100,7 +100,7 @@ jobs:
           ref: ${{ github.event.client_payload.ref }}
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@v2.23.2
 
       - uses: actions/upload-artifact@v4
         with:

Makefile

@@ -31,7 +31,7 @@ DEVPKGS=-rdev-requirements.txt -rtest-requirements.txt -rmypy-requirements.txt -
 COVBASE=coverage run --append
 PYTEST_EXTRA ?=
 
-VERSION=8.8.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
+VERSION=8.9.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
 
 ## all                    : default task (install schema-salad in dev mode)

cibw-requirements.txt

@@ -1 +1 @@
-cibuildwheel==2.22.0
+cibuildwheel==2.23.2

schema_salad/ref_resolver.py

@@ -14,6 +14,7 @@
 import requests
 from cachecontrol.caches import SeparateBodyFileCache
 from cachecontrol.wrapper import CacheControl
+from mypy_extensions import mypyc_attr
 from rdflib.exceptions import ParserError
 from rdflib.graph import Graph
 from rdflib.namespace import OWL, RDF, RDFS
@@ -134,6 +135,11 @@ def SubLoader(loader: "Loader") -> "Loader":
     )
 
 
+def _url_norm(url: str) -> str:
+    return urllib.parse.urlsplit(url).geturl()
+
+
+@mypyc_attr(allow_interpreted_subclasses=True)
 class Loader:
     def __init__(
         self,
@@ -150,9 +156,7 @@ def __init__(
         doc_cache: Union[str, bool] = True,
         salad_version: Optional[str] = None,
     ) -> None:
-        self.idx: IdxType = (
-            NormDict(lambda url: urllib.parse.urlsplit(url).geturl()) if idx is None else idx
-        )
+        self.idx: IdxType = NormDict(_url_norm) if idx is None else idx
 
         self.ctx: ContextType = {}
         self.graph = schemagraph if schemagraph is not None else Graph()

schema_salad/tests/test_avro_names.py

@@ -9,7 +9,6 @@
 def test_avro_loading() -> None:
     """Confirm conversion of SALAD style names to avro."""
     path = get_data("tests/test_schema/avro_naming.yml")
-    assert path
     document_loader, avsc_names, schema_metadata, metaschema_loader = load_schema(path)
     assert isinstance(avsc_names, Names)
     assert avsc_names.get_name("com.example.derived_schema.ExtendedThing", None)

schema_salad/tests/test_cg.py

@@ -1,16 +1,14 @@
 import json
-import os
 from typing import Any
 
 import pytest
 
 import schema_salad.metaschema as cg_metaschema
 from schema_salad.exceptions import ValidationException
-from schema_salad.ref_resolver import file_uri
 from schema_salad.utils import yaml_no_ts
 
 from .matcher import JsonDiffMatcher
-from .util import get_data
+from .util import get_data_uri, get_path
 
 
 def test_load() -> None:
@@ -45,12 +43,11 @@ def test_err() -> None:
 
 def test_include() -> None:
     doc = {"name": "hello", "doc": [{"$include": "hello.txt"}], "type": "documentation"}
-    path = get_data("tests/_")
-    assert path
+    path_uri = get_data_uri("tests") + "/_"
     rf = cg_metaschema.Documentation.fromDoc(
         doc,
         "http://example.com/",
-        cg_metaschema.LoadingOptions(fileuri=file_uri(path), no_link_check=True),
+        cg_metaschema.LoadingOptions(fileuri=path_uri, no_link_check=True),
     )
     assert "http://example.com/#hello" == rf.name
     assert ["hello world!\n"] == rf.doc
@@ -64,9 +61,7 @@ def test_include() -> None:
 
 def test_import() -> None:
     doc = {"type": "record", "fields": [{"$import": "hellofield.yml"}]}
-    tests_path = get_data("tests")
-    assert tests_path
-    lead = file_uri(os.path.normpath(tests_path))
+    lead = get_data_uri("tests")
     rs = cg_metaschema.RecordSchema.fromDoc(
         doc, "http://example.com/", cg_metaschema.LoadingOptions(fileuri=lead + "/_")
     )
@@ -87,11 +82,9 @@ def test_import() -> None:
 
 
 def test_import2() -> None:
-    path = get_data("tests/docimp/d1.yml")
-    assert path
-    rs = cg_metaschema.load_document(file_uri(path), "", cg_metaschema.LoadingOptions())
-    path2 = get_data("tests/docimp/d1.yml")
-    assert path2
+    path_uri = get_data_uri("tests/docimp/d1.yml")
+    rs = cg_metaschema.load_document(path_uri, "", cg_metaschema.LoadingOptions())
+    path2_uri = get_data_uri("tests/docimp/d1.yml")
     assert [
         {
             "doc": [
@@ -104,7 +97,7 @@ def test_import2() -> None:
                 "hello 5",
             ],
             "type": "documentation",
-            "name": file_uri(path2) + "#Semantic_Annotations_for_Linked_Avro_Data",
+            "name": path2_uri + "#Semantic_Annotations_for_Linked_Avro_Data",
         }
     ] == [r.save() for r in rs]
 
@@ -158,10 +151,9 @@ def test_idmap2() -> None:
 
 
 def test_load_pt() -> None:
-    path = get_data("tests/pt.yml")
-    assert path
+    path_uri = get_data_uri("tests/pt.yml")
     doc = cg_metaschema.load_document(
-        file_uri(path), "", cg_metaschema.LoadingOptions(no_link_check=True)
+        path_uri, "", cg_metaschema.LoadingOptions(no_link_check=True)
     )
     assert [
         "https://w3id.org/cwl/salad#null",
@@ -187,18 +179,15 @@ def test_shortname() -> None:
 @pytest.fixture
 def metaschema_pre() -> Any:
     """Prep-parsed schema for testing."""
-    path2 = get_data("tests/metaschema-pre.yml")
-    assert path2
-    with open(path2) as f:
+    with get_path("tests/metaschema-pre.yml").open() as f:
         pre = json.load(f)
     return pre
 
 
 def test_load_metaschema(metaschema_pre: Any) -> None:
-    path = get_data("metaschema/metaschema.yml")
-    assert path
+    path_uri = get_data_uri("metaschema/metaschema.yml")
     doc = cg_metaschema.load_document(
-        file_uri(path),
+        path_uri,
         "",
         cg_metaschema.LoadingOptions(no_link_check=True),
     )
@@ -209,33 +198,30 @@ def test_load_metaschema(metaschema_pre: Any) -> None:
 
 
 def test_load_by_yaml_metaschema(metaschema_pre: Any) -> None:
-    path = get_data("metaschema/metaschema.yml")
-    assert path
-    with open(path) as path_handle:
+    path = get_path("metaschema/metaschema.yml")
+    with path.open() as path_handle:
         yaml = yaml_no_ts()
         yaml_doc = yaml.load(path_handle)
     doc = cg_metaschema.load_document_by_yaml(
         yaml_doc,
-        file_uri(path),
+        path.as_uri(),
         None,
     )
     saved = [d.save(relative_uris=False) for d in doc]
     assert saved == JsonDiffMatcher(metaschema_pre)
 
 
 def test_load_cwlschema() -> None:
-    path = get_data("tests/test_schema/CommonWorkflowLanguage.yml")
-    assert path
+    path_uri = get_data_uri("tests/test_schema/CommonWorkflowLanguage.yml")
     doc = cg_metaschema.load_document(
-        file_uri(path),
+        path_uri,
         "",
         cg_metaschema.LoadingOptions(no_link_check=True),
     )
-    path2 = get_data("tests/cwl-pre.yml")
-    assert path2
+    path2 = get_path("tests/cwl-pre.yml")
     saved = [d.save(relative_uris=False) for d in doc]
-    # with open(path2, "w") as f:
+    # with path2.open("w") as f:
     #     json.dump(saved, f, indent=2)
-    with open(path2) as f:
+    with path2.open() as f:
         pre = json.load(f)
     assert saved == JsonDiffMatcher(pre)

schema_salad/tests/test_codegen_errors.py

@@ -3,8 +3,7 @@
 import importlib
 from collections.abc import MutableSequence
 from pathlib import Path
-from typing import Any, Optional, Union, cast
-from urllib.parse import urlparse
+from typing import Any, Optional, cast
 
 import pytest
 
@@ -14,7 +13,7 @@
 from schema_salad.schema import load_schema
 from schema_salad.utils import yaml_no_ts
 
-from .util import cwl_file_uri, get_data
+from .util import cwl_file_uri, get_path
 
 
 def test_error_message1(tmp_path: Path) -> None:
@@ -23,17 +22,17 @@ def test_error_message1(tmp_path: Path) -> None:
 \s+\*\s+missing\s+required\s+field\s+`inputs`
 \s+\*\s+missing\s+required\s+field\s+`outputs`
 \s+\*\s+missing\s+required\s+field\s+`steps`"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
 
 def test_error_message2(tmp_path: Path) -> None:
     t = "test_schema/test2.cwl"
     match = r"""^.*test2\.cwl:2:1:\s+Field\s+`class`\s+contains\s+undefined\s+reference\s+to\s+`file://.+/schema_salad/tests/test_schema/xWorkflow`$"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
@@ -46,8 +45,8 @@ def test_error_message4(tmp_path: Path) -> None:
 .*test4\.cwl:7:3:\s+checking\s+object\s+`.*test4\.cwl#bar`\s+using\s+`WorkflowOutputParameter`
 \s+the\s+`type`\s+field\s+is\s+not\s+valid\s+because:
 \s+Value\s+`12`\s+is\s+a\s+int,\s+but\s+valid\s+types\s+for\s+this\s+field\s+are\s+\((str|object),\s+(str|object)\)"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
@@ -58,8 +57,8 @@ def test_error_message5(tmp_path: Path) -> None:
 .+test5\.cwl:8:1:\s+the\s+`steps`\s+field\s+is\s+not\s+valid\s+because:
 .+test5\.cwl:8:9:\s+array\s+item\s+is\s+invalid\s+because
 \s+Value\s+is\s+a\s+int,\s+but\s+valid\s+type\s+for\s+this\s+field\s+is\s+an\s+object"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
@@ -72,8 +71,8 @@ def test_error_message6(tmp_path: Path) -> None:
 \s+missing\s+required\s+field\s+`class`
 +\*\s+tried\s+`Workflow`\s+but
 \s+missing\s+required\s+field\s+`class`"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
@@ -86,8 +85,8 @@ def test_error_message7(tmp_path: Path) -> None:
 .*test7\.cwl:9:3:\s+checking\s+object\s+`.*test7.cwl#step1`\s+using\s+`WorkflowStep`
 \s+\*\s+missing\s+required\s+field\s+`run`
 .*test7\.cwl:10:5:\s+\*\s+invalid\s+field\s+`scatter_method`,\s+expected\s+one\s+of:\s+.*\s+.*\s+.*\s+.*\s+.*\s+.*\s+.*\s+.*\s+.*\s+.*$"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
@@ -101,8 +100,8 @@ def test_error_message8(tmp_path: Path) -> None:
 \s+\*\s+missing\s+required\s+field\s+`run`
 .*test8\.cwl:10:5:\s+\*\s+the\s+`scatterMethod`\s+field\s+is\s+not\s+valid\s+because:
 \s+contains\s+undefined\s+reference\s+to\s+`file:///.*/tests/test_schema/abc`$"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
@@ -116,8 +115,8 @@ def test_error_message9(tmp_path: Path) -> None:
 \s+\*\s+missing\s+required\s+field\s+`run`
 .*test9\.cwl:10:5:\s+\*\s+the\s+`scatterMethod`\s+field\s+is\s+not\s+valid\s+because:
 \s+Value\s+`12`\s+is\s+a\s+int,\s+but\s+valid\s+values\s+for\s+this\s+field\s+are\s+\("(dotproduct|nested_crossproduct|flat_crossproduct)",\s+"(dotproduct|nested_crossproduct|flat_crossproduct)",\s+"(dotproduct|nested_crossproduct|flat_crossproduct)"\)"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
@@ -131,8 +130,8 @@ def test_error_message10(tmp_path: Path) -> None:
 \s+\*\s+missing\s+required\s+field\s+`run`
 .*test10\.cwl:10:5:\s+\*\s+the\s+`scatterMethod`\s+field\s+is\s+not\s+valid\s+because:
 \s+Value\s+is\s+a\s+array,\s+but\s+valid\s+types\s+for\s+this\s+field\s+are\s+\("dotproduct|nested_crossproduct|flat_crossproduct",\s+"dotproduct|nested_crossproduct|flat_crossproduct",\s+"dotproduct|nested_crossproduct|flat_crossproduct"\)"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
@@ -144,8 +143,8 @@ def test_error_message11(tmp_path: Path) -> None:
 \s+array\s+item\s+is\s+invalid\s+because
 .*test11\.cwl:9:3:\s+checking\s+object\s+`.*test11\.cwl#step1`\s+using\s+`WorkflowStep`
 .*test11\.cwl:10:5:\s+the\s+`run`\s+field\s+is\s+not\s+valid\s+because:\s+contains\s+undefined\s+reference\s+to\s+`file:///.*/tests/test_schema/blub\.cwl`$"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
@@ -161,13 +160,13 @@ def test_error_message15(tmp_path: Path) -> None:
 \s+tried\s+`CommandLineBinding`\s+but
 .*test15\.cwl:11:7:\s+\*\s+invalid\s+field\s+`invalid_field`,\s+expected\s+one\s+of:\s+`loadContents`,\s+`position`,\s+`prefix`,\s+`separate`,\s+`itemSeparator`,\s+`valueFrom`,\s+`shellQuote`
 .*test15\.cwl:12:7:\s+\*\s+invalid\s+field\s+`another_invalid_field`,\s+expected one\s+of:\s+`loadContents`,\s+`position`,\s+`prefix`,\s+`separate`,\s+`itemSeparator`,\s+`valueFrom`,\s+`shellQuote`$"""
-    path = get_data("tests/" + t)
-    assert path
+    path = get_path("tests/" + t)
+    assert path.exists()
     with pytest.raises(ValidationException, match=match):
         load_document_by_uri(tmp_path, path)
 
 
-def load_document_by_uri(tmp_path: Path, path: Union[str, Path]) -> Any:
+def load_document_by_uri(tmp_path: Path, path: Path) -> Any:
     src_target = tmp_path / "cwl_v1_0.py"
     python_codegen(cwl_file_uri, src_target)
     spec = importlib.util.spec_from_file_location("cwl_v1_0", src_target)
@@ -177,22 +176,15 @@ def load_document_by_uri(tmp_path: Path, path: Union[str, Path]) -> Any:
     spec.loader.exec_module(temp_cwl_v1_0)
     cwl_v1_0: Any = temp_cwl_v1_0
 
-    if isinstance(path, str):
-        uri = urlparse(path)
-        if not uri.scheme or uri.scheme == "file":
-            real_path = Path(uri.path).resolve().as_uri()
-        else:
-            real_path = path
-    else:
-        real_path = path.resolve().as_uri()
+    path_uri = path.resolve().as_uri()
 
-    baseuri = str(real_path)
+    baseuri = path_uri
 
     loadingOptions = cwl_v1_0.LoadingOptions(fileuri=baseuri)
 
     with open(path) as file:
         doc = file.read()
-    # doc = loadingOptions.fetcher.fetch_text(urllib.parse.unquote(str(real_path)))
+    # doc = loadingOptions.fetcher.fetch_text(urllib.parse.unquote(path_uri))
     yaml = yaml_no_ts()
     doc = yaml.load(doc)