better pass --skip-schemas

Author: mr-c

cwltool/context.py

@@ -120,6 +120,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         self.codegen_idx: Dict[str, Tuple[Any, "LoadingOptions"]] = {}
         self.fast_parser = False
         self.skip_resolve_all = False
+        self.skip_schemas = False
 
         super().__init__(kwargs)
 

cwltool/load_tool.py

@@ -22,6 +22,7 @@
 
 from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils
 from schema_salad.exceptions import ValidationException
+from schema_salad.fetcher import Fetcher
 from schema_salad.ref_resolver import Loader, file_uri
 from schema_salad.schema import validate_doc
 from schema_salad.sourceline import SourceLine, cmap
@@ -32,7 +33,6 @@
     ResolveType,
     json_dumps,
 )
-from schema_salad.fetcher import Fetcher
 
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
@@ -395,7 +395,6 @@ def resolve_and_validate_document(
     workflowobj: Union[CommentedMap, CommentedSeq],
     uri: str,
     preprocess_only: bool = False,
-    skip_schemas: Optional[bool] = None,
 ) -> Tuple[LoadingContext, str]:
     """Validate a CWL document."""
     if not loadingContext.loader:
@@ -502,7 +501,7 @@ def resolve_and_validate_document(
         idx=loader.idx,
         cache=sch_document_loader.cache,
         fetcher_constructor=loadingContext.fetcher_constructor,
-        skip_schemas=skip_schemas,
+        skip_schemas=loadingContext.skip_schemas,
         doc_cache=loadingContext.doc_cache,
     )
 
@@ -652,7 +651,9 @@ def load_tool(
     loadingContext, workflowobj, uri = fetch_document(argsworkflow, loadingContext)
 
     loadingContext, uri = resolve_and_validate_document(
-        loadingContext, workflowobj, uri
+        loadingContext,
+        workflowobj,
+        uri,
     )
 
     return make_tool(uri, loadingContext)
@@ -682,15 +683,13 @@ def recursive_resolve_and_validate_document(
     workflowobj: Union[CommentedMap, CommentedSeq],
     uri: str,
     preprocess_only: bool = False,
-    skip_schemas: Optional[bool] = None,
 ) -> Tuple[LoadingContext, str, Process]:
     """Validate a CWL document, checking that a tool object can be built."""
     loadingContext, uri = resolve_and_validate_document(
         loadingContext,
         workflowobj,
         uri,
         preprocess_only=preprocess_only,
-        skip_schemas=skip_schemas,
     )
     tool = make_tool(uri, loadingContext)
     return loadingContext, uri, tool

cwltool/main.py

@@ -1136,7 +1136,6 @@ def main(
                 workflowobj,
                 uri,
                 preprocess_only=(args.print_pre or args.pack),
-                skip_schemas=args.skip_schemas,
             )
 
             if loadingContext.loader is None:

tests/test_misc_cli.py

@@ -67,3 +67,23 @@ def test_error_graph_with_no_default() -> None:
         "Tool file contains graph of multiple objects, must specify one of #echo, #cat, #collision"
         in stderr
     )
+
+
+def test_skip_schemas_external_step() -> None:
+    """Test that --skip-schemas works even for bad schemas in external docs."""
+    exit_code, stdout, stderr = get_main_output(
+        [
+            "--print-rdf",
+            "--skip-schemas",
+            get_data("tests/wf/revsort_step_bad_schema.cwl"),
+        ]
+    )
+    assert exit_code == 0
+    assert (
+        "Repeat node-elements inside property elements: "
+        "http://www.w3.org/1999/xhtmlmeta"
+    ) not in stderr
+    assert (
+        "Could not load extension schema https://bad.example.com/missing.ttl: "
+        "Error fetching https://bad.example.com/missing.ttl"
+    ) not in stderr

tests/wf/revsort_step_bad_schema.cwl

@@ -0,0 +1,73 @@
+#!/usr/bin/env cwl-runner
+#
+# This is a two-step workflow which uses "revtool" and "sorttool" defined above.
+#
+class: Workflow
+doc: "Reverse the lines in a document, then sort those lines."
+cwlVersion: v1.0
+
+# Requirements & hints specify prerequisites and extensions to the workflow.
+# In this example, DockerRequirement specifies a default Docker container
+# in which the command line tools will execute.
+hints:
+  - class: DockerRequirement
+    dockerPull: docker.io/debian:stable-slim
+
+
+# The inputs array defines the structure of the input object that describes
+# the inputs to the workflow.
+#
+# The "reverse_sort" input parameter demonstrates the "default" field.  If the
+# field "reverse_sort" is not provided in the input object, the default value will
+# be used.
+inputs:
+  workflow_input:
+    type: File
+    doc: "The input file to be processed."
+    format: iana:text/plain
+    default:
+      class: File
+      location: hello.txt
+  reverse_sort:
+    type: boolean
+    default: true
+    doc: "If true, reverse (descending) sort"
+
+# The "outputs" array defines the structure of the output object that describes
+# the outputs of the workflow.
+#
+# Each output field must be connected to the output of one of the workflow
+# steps using the "outputSource" field.  Here, the parameter "sorted_output" of the
+# workflow comes from the "sorted_output" output of the "sorted" step.
+outputs:
+  sorted_output:
+    type: File
+    outputSource: sorted/sorted_output
+    doc: "The output with the lines reversed and sorted."
+
+# The "steps" array lists the executable steps that make up the workflow.
+# The tool to execute each step is listed in the "run" field.
+#
+# In the first step, the "in" field of the step connects the upstream
+# parameter "workflow_input" of the workflow to the input parameter of the tool
+# "revtool_input"
+#
+# In the second step, the "in" field of the step connects the output
+# parameter "revtool_output" from the first step to the input parameter of the
+# tool "sorted_input".
+steps:
+  rev:
+    in:
+      revtool_input: workflow_input
+    out: [revtool_output]
+    run: revtool_bad_schema.cwl
+
+  sorted:
+    in:
+      sorted_input: rev/revtool_output
+      reverse: reverse_sort
+    out: [sorted_output]
+    run: sorttool.cwl
+
+$namespaces:
+  iana: https://www.iana.org/assignments/media-types/

tests/wf/revtool_bad_schema.cwl

@@ -0,0 +1,42 @@
+#!/usr/bin/env cwl-runner
+#
+# Simplest example command line program wrapper for the Unix tool "rev".
+#
+class: CommandLineTool
+cwlVersion: v1.0
+doc: "Reverse each line using the `rev` command"
+$schemas:
+  - https://bad.example.com/missing.ttl
+  - https://schema.org/docs/
+  - https://schema.org/docs/schema_org_rdfa.html
+
+# The "inputs" array defines the structure of the input object that describes
+# the inputs to the underlying program.  Here, there is one input field
+# defined that will be called "input" and will contain a "File" object.
+#
+# The input binding indicates that the input value should be turned into a
+# command line argument.  In this example inputBinding is an empty object,
+# which indicates that the file name should be added to the command line at
+# a default location.
+inputs:
+  revtool_input:
+    type: File
+    inputBinding: {}
+
+# The "outputs" array defines the structure of the output object that
+# describes the outputs of the underlying program.  Here, there is one
+# output field defined that will be called "output", must be a "File" type,
+# and after the program executes, the output value will be the file
+# output.txt in the designated output directory.
+outputs:
+  revtool_output:
+    type: File
+    outputBinding:
+      glob: output.txt
+
+# The actual program to execute.
+baseCommand: rev
+
+# Specify that the standard output stream must be redirected to a file called
+# output.txt in the designated output directory.
+stdout: output.txt