Skip to content

Commit a166e70

Browse files
Zac-HDZac-HD
committed
Clarify test marker lists
1 parent 82563f9 commit a166e70

File tree

1 file changed

+116
-52
lines changed

1 file changed

+116
-52
lines changed

‎tests/test_from_schema.py‎

Lines changed: 116 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -64,47 +64,53 @@ def test_invalid_schemas_raise(schema):
6464
from_schema(schema).example()
6565

6666

67+
INVALID_SCHEMAS = {
68+
# Includes a list where it should have a dict
69+
"TypeScript Lint configuration file",
70+
# This schema is missing the "definitions" key which means they're not resolvable.
71+
"Cirrus CI configuration files",
72+
# Empty list for requires, which is invalid
73+
"Release Drafter configuration file",
74+
# Many, many schemas have invalid $schema keys, which emit a warning (-Werror)
75+
"A JSON schema for CRYENGINE projects (.cryproj files)",
76+
"JSDoc configuration file",
77+
"Meta-validation schema for JSON Schema Draft 8",
78+
"Static Analysis Results Format (SARIF) External Property File Format, Version 2.1.0-rtm.2",
79+
"Static Analysis Results Format (SARIF) External Property File Format, Version 2.1.0-rtm.3",
80+
"Static Analysis Results Format (SARIF) External Property File Format, Version 2.1.0-rtm.4",
81+
"Static Analysis Results Format (SARIF) External Property File Format, Version 2.1.0-rtm.5",
82+
"Static Analysis Results Format (SARIF), Version 2.1.0-rtm.2",
83+
"Zuul CI configuration file",
84+
}
85+
UNSUPPORTED_SCHEMAS = {
86+
# Technically valid, but using regex patterns not supported by Python
87+
"draft7/ECMA 262 regex escapes control codes with \\c and lower letter",
88+
"draft7/ECMA 262 regex escapes control codes with \\c and upper letter",
89+
}
6790
FLAKY_SCHEMAS = {
91+
# The following schemas refer to an `$id` rather than a JSON pointer.
92+
# This is valid, but not supported by the Python library - see e.g.
93+
# https://json-schema.org/understanding-json-schema/structuring.html#using-id-with-ref
94+
"draft4/Location-independent identifier",
95+
"draft7/Location-independent identifier",
6896
# Yep, lists of lists of lists of lists of lists of integers are HealthCheck-slow
97+
# TODO: write a separate test with healthchecks disabled?
6998
"draft4/nested items",
7099
"draft7/nested items",
71100
"draft4/oneOf with missing optional property",
72101
"draft7/oneOf with missing optional property",
73-
# Something weird about a null that should be a string?? TODO: debug that.
74-
"Datalogic Scan2Deploy Android file",
75-
"Datalogic Scan2Deploy CE file",
76-
# Schema requires draft 03, which hypothesis-jsonschema doesn't support
77-
"A JSON Schema for ninjs by the IPTC. News and publishing information. See https://iptc.org/standards/ninjs/-1.0",
78-
# Just not handling this one correctly yet
102+
# Sometimes unsatisfiable. TODO: improve canonicalisation to remove filters
103+
"Drone CI configuration file",
104+
"PHP Composer configuration file",
105+
"Pyrseas database schema versioning for Postgres databases, v0.8",
106+
# Apparently we're not handling this one correctly?
79107
"draft4/additionalProperties should not look in applicators",
80108
"draft7/additionalProperties should not look in applicators",
81-
"draft7/ECMA 262 regex escapes control codes with \\c and lower letter",
82-
"draft7/ECMA 262 regex escapes control codes with \\c and upper letter",
83-
# Reference-related
84-
"draft4/remote ref, containing refs itself",
85-
# Occasionally really slow
109+
}
110+
SLOW_SCHEMAS = {
86111
"snapcraft project (https://snapcraft.io)",
87112
"batect configuration file",
88113
"UI5 Tooling Configuration File (ui5.yaml)",
89-
# Sometimes unsatisfiable. TODO: improve canonicalisation to remove filters
90-
"Drone CI configuration file",
91-
"PHP Composer configuration file",
92-
"Pyrseas database schema versioning for Postgres databases, v0.8",
93-
# This schema is missing the "definitions" key which means they're not resolvable.
94-
"Cirrus CI configuration files",
95-
# These schemas use remote references without a URL schema, which is invalid
96-
"A JSON schema for a Dolittle bounded context's resource configurations",
97-
"A JSON schema for Dolittle application's bounded context configuration",
98-
# The following schemas refer to an `$id` rather than a JSON pointer.
99-
# This is valid, but not supported by the Python library - see e.g.
100-
# https://json-schema.org/understanding-json-schema/structuring.html#using-id-with-ref
101-
"draft4/Location-independent identifier",
102-
"draft7/Location-independent identifier",
103-
# Bad reference forgot the leading "#"
104-
"Microsoft Briefcase configuration file",
105-
# Includes a list where it should have a dict
106-
"TypeScript Lint configuration file",
107-
# These schemas got very, very slow after we fixed object properties merging
108114
"Renovate config file (https://github.com/renovatebot/renovate)",
109115
"Renovate config file (https://renovatebot.com/)",
110116
"Jenkins X Pipeline YAML configuration files",
@@ -113,7 +119,18 @@ def test_invalid_schemas_raise(schema):
113119
"Configuration file for stylelint",
114120
"Travis CI configuration file",
115121
"JSON schema for ESLint configuration files",
116-
# TODO: work out how to use slow marker here
122+
"Ansible task files-2.0",
123+
"Ansible task files-2.1",
124+
"Ansible task files-2.2",
125+
"Ansible task files-2.3",
126+
"Ansible task files-2.4",
127+
"Ansible task files-2.5",
128+
"Ansible task files-2.6",
129+
"Ansible task files-2.7",
130+
"Ansible task files-2.9",
131+
# oneOf on property names means only objects are valid, but it's a very
132+
# filter-heavy way to express that. TODO: canonicalise oneOf to anyOf.
133+
"draft7/oneOf complex types",
117134
}
118135

119136
with open(Path(__file__).parent / "corpus-schemastore-catalog.json") as f:
@@ -128,38 +145,86 @@ def test_invalid_schemas_raise(schema):
128145

129146
def to_name_params(corpus):
130147
for n in sorted(corpus):
131-
if n.endswith("/oneOf complex types"):
132-
# oneOf on property names means only objects are valid,
133-
# but it's a very filter-heavy way to express that...
134-
# TODO: see if we can auto-detect this, fix it, and emit a warning.
135-
assert "type" not in corpus[n]
136-
corpus[n]["type"] = "object"
137-
if n in FLAKY_SCHEMAS or n.startswith("Ansible task files-"):
148+
if n in INVALID_SCHEMAS:
149+
continue
150+
if n in UNSUPPORTED_SCHEMAS:
151+
continue
152+
elif n in SLOW_SCHEMAS:
138153
yield pytest.param(n, marks=pytest.mark.skip)
154+
elif n in FLAKY_SCHEMAS:
155+
yield pytest.param(n, marks=pytest.mark.skip(strict=False))
139156
else:
140157
if isinstance(corpus[n], dict) and "$schema" in corpus[n]:
141-
try:
142-
jsonschema.validators.validator_for(corpus[n]).check_schema(
143-
corpus[n]
144-
)
145-
except Exception:
146-
# The metaschema specified by $schema was not found.
147-
yield pytest.param(n, marks=pytest.mark.skip)
148-
continue
158+
jsonschema.validators.validator_for(corpus[n]).check_schema(corpus[n])
149159
yield n
150160

151161

162+
RECURSIVE_REFS = {
163+
# From upstream validation test suite
164+
"draft4/valid definition",
165+
"draft4/remote ref, containing refs itself",
166+
"draft7/remote ref, containing refs itself",
167+
"draft7/root pointer ref",
168+
"draft7/valid definition",
169+
# Schema also requires draft 03, which hypothesis-jsonschema doesn't support
170+
"A JSON Schema for ninjs by the IPTC. News and publishing information. See https://iptc.org/standards/ninjs/-1.0",
171+
# From schemastore
172+
"A JSON schema for Open API documentation files",
173+
"Avro Schema Avsc file",
174+
"AWS CloudFormation provides a common language for you to describe and provision all the infrastructure resources in your cloud environment.",
175+
"JSON schema .NET template files",
176+
"AppVeyor CI configuration file",
177+
"JSON Document Transofrm",
178+
"JSON Linked Data files",
179+
"Meta-validation schema for JSON Schema Draft 4",
180+
"JSON schema for vim plugin addon-info.json metadata files",
181+
"Meta-validation schema for JSON Schema Draft 7",
182+
"Neotys as-code load test specification, more at: https://github.com/Neotys-Labs/neoload-cli",
183+
"Metadata spec v1.26.4 for KSP-CKAN",
184+
"Digital Signature Service Core Protocols, Elements, and Bindings Version 2.0",
185+
"Opctl schema for describing an op",
186+
"Metadata spec v1.27 for KSP-CKAN",
187+
"PocketMine plugin manifest file",
188+
"BuckleScript configuration file",
189+
"Schema for CircleCI 2.0 config files",
190+
"Source Map files version 3",
191+
"Schema for Minecraft Bukkit plugin description files",
192+
"Swagger API 2.0 schema",
193+
"Static Analysis Results Interchange Format (SARIF) version 1",
194+
"Static Analysis Results Format (SARIF), Version 2.1.0-rtm.4",
195+
"Static Analysis Results Interchange Format (SARIF) version 2",
196+
"Static Analysis Results Format (SARIF), Version 2.1.0-rtm.3",
197+
"Static Analysis Results Format (SARIF), Version 2.1.0-rtm.5",
198+
"Web component file",
199+
"Vega visualization specification file",
200+
"The AWS Serverless Application Model (AWS SAM, previously known as Project Flourish) extends AWS CloudFormation to provide a simplified way of defining the Amazon API Gateway APIs, AWS Lambda functions, and Amazon DynamoDB tables needed by your serverless application.",
201+
"Windows App localization file",
202+
"YAML schema for GitHub Workflow",
203+
"JSON-stat 2.0 Schema",
204+
"Vega-Lite visualization specification file",
205+
"Language grammar description files in Textmate and compatible editors",
206+
"JSON Schema for GraphQL Mesh Config gile-0.0.16",
207+
}
208+
209+
152210
def xfail_on_reference_resolve_error(f):
153211
@proxies(f)
154212
def inner(*args, **kwargs):
213+
_, name = args
214+
assert isinstance(name, str)
155215
try:
156-
return f(*args, **kwargs)
216+
f(*args, **kwargs)
217+
assert name not in RECURSIVE_REFS
157218
except jsonschema.exceptions.RefResolutionError as err:
158-
# Currently: could be recursive, remote, or incompatible base schema
159-
if isinstance(err, HypothesisRefResolutionError) or isinstance(
160-
err._cause, HypothesisRefResolutionError
219+
if (
220+
isinstance(err, HypothesisRefResolutionError)
221+
or isinstance(err._cause, HypothesisRefResolutionError)
222+
) and (
223+
"does not fetch remote references" in str(err)
224+
or name in RECURSIVE_REFS
225+
and "Could not resolve recursive references" in str(err)
161226
):
162-
pytest.xfail("Could not resolve a reference")
227+
pytest.xfail()
163228
raise
164229

165230
return inner
@@ -193,7 +258,6 @@ def test_can_generate_for_test_suite_schema(data, name):
193258

194259

195260
@pytest.mark.parametrize("name", to_name_params(invalid_suite))
196-
@xfail_on_reference_resolve_error
197261
def test_cannot_generate_for_empty_test_suite_schema(name):
198262
strat = from_schema(invalid_suite[name])
199263
with pytest.raises(Exception):

0 commit comments

Comments
 (0)