Skip to content

Commit bfd7e5d

Browse files
BethanyGBethanyG
authored
Changed sort order for concept exercises with failed subtests and rerecoreded golden output for them, (#122)
1 parent 2095e20 commit bfd7e5d

File tree

11 files changed

+462
-153
lines changed

11 files changed

+462
-153
lines changed

‎runner/data.py‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -153,15 +153,31 @@ def _factory(items):
153153

154154
def as_json(self):
155155
"""
156-
Trim off the TestClass name and test_ prefix from each test_name.
157-
Replace underscores with spaces for more human-readable strings.
158-
Sort the current tests array by task_id and then Dump all results to formatted JSON.
156+
- Trim off the TestClass name and test_ prefix from each test_name.
157+
- Replace underscores with spaces for more human-readable strings.
158+
- Add a sort order signifier (~) to parent tests with subtests.
159+
(~) ensures parent tests sort to last position.
160+
161+
- If it is a concept exercise, sort the current tests array by
162+
task_id and variation# then Dump all results to formatted JSON.
159163
"""
164+
165+
160166
trim_name = compile(r'^(.+)(Test\.test_)')
161167
results = asdict(self, dict_factory=self._factory)
168+
concept_exercise = False
162169

163170
for item in results["tests"]:
164-
item["name"] = sub(trim_name, '\\1 > ', item["name"]).replace('_', ' ')
165171

166-
results["tests"] = sorted(results["tests"], key= lambda item: item["task_id"])
172+
if "[variation" not in item["name"] and item["task_id"] > 0:
173+
item["name"] = sub(trim_name, '\\1 > ', item["name"]).replace('_', ' ') + "~"
174+
concept_exercise = True
175+
else:
176+
item["name"] = sub(trim_name, '\\1 > ', item["name"]).replace('_', ' ')
177+
178+
if concept_exercise:
179+
results["tests"] = sorted(results["tests"], key= lambda item: (item["task_id"], item["name"]))
180+
else:
181+
results["tests"] = sorted(results["tests"], key=lambda item: (item["task_id"]))
182+
167183
return dumps(results, indent=2)

‎test/example-all-fail-tasks-and-subtests/results.json‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2,48 +2,6 @@
22
"version": 3,
33
"status": "fail",
44
"tests": [
5-
{
6-
"name": "ExampleAllFail > abc",
7-
"status": "fail",
8-
"message": "One or more variations of this test failed. Details can be found under each [variant#].",
9-
"test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
10-
"task_id": 1
11-
},
12-
{
13-
"name": "ExampleAllFail > hello",
14-
"status": "fail",
15-
"message": "One or more variations of this test failed. Details can be found under each [variant#].",
16-
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
17-
"task_id": 1
18-
},
19-
{
20-
"name": "ExampleAllFail > hello [variation #1]",
21-
"status": "fail",
22-
"message": "AssertionError: 'Goodbye!' != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.",
23-
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
24-
"task_id": 1
25-
},
26-
{
27-
"name": "ExampleAllFail > hello [variation #2]",
28-
"status": "fail",
29-
"message": "AssertionError: 'Goodbye!' != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.",
30-
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
31-
"task_id": 1
32-
},
33-
{
34-
"name": "ExampleAllFail > hello [variation #3]",
35-
"status": "fail",
36-
"message": "AssertionError: 'Goodbye!' != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.",
37-
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
38-
"task_id": 1
39-
},
40-
{
41-
"name": "ExampleAllFail > hello [variation #4]",
42-
"status": "fail",
43-
"message": "AssertionError: 'Goodbye!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.",
44-
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
45-
"task_id": 1
46-
},
475
{
486
"name": "ExampleAllFail > abc [variation #1]",
497
"status": "fail",
@@ -101,18 +59,46 @@
10159
"task_id": 1
10260
},
10361
{
104-
"name": "ExampleAllFailOther > dummy",
62+
"name": "ExampleAllFail > abc~",
10563
"status": "fail",
10664
"message": "One or more variations of this test failed. Details can be found under each [variant#].",
10765
"test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
108-
"task_id": 2
66+
"task_id": 1
67+
},
68+
{
69+
"name": "ExampleAllFail > hello [variation #1]",
70+
"status": "fail",
71+
"message": "AssertionError: 'Goodbye!' != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.",
72+
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
73+
"task_id": 1
74+
},
75+
{
76+
"name": "ExampleAllFail > hello [variation #2]",
77+
"status": "fail",
78+
"message": "AssertionError: 'Goodbye!' != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.",
79+
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
80+
"task_id": 1
10981
},
11082
{
111-
"name": "ExampleAllFailOther > hello",
83+
"name": "ExampleAllFail > hello [variation #3]",
84+
"status": "fail",
85+
"message": "AssertionError: 'Goodbye!' != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.",
86+
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
87+
"task_id": 1
88+
},
89+
{
90+
"name": "ExampleAllFail > hello [variation #4]",
91+
"status": "fail",
92+
"message": "AssertionError: 'Goodbye!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.",
93+
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
94+
"task_id": 1
95+
},
96+
{
97+
"name": "ExampleAllFail > hello~",
11298
"status": "fail",
11399
"message": "One or more variations of this test failed. Details can be found under each [variant#].",
114-
"test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
115-
"task_id": 2
100+
"test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
101+
"task_id": 1
116102
},
117103
{
118104
"name": "ExampleAllFailOther > dummy [variation #1]",
@@ -170,6 +156,13 @@
170156
"test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
171157
"task_id": 2
172158
},
159+
{
160+
"name": "ExampleAllFailOther > dummy~",
161+
"status": "fail",
162+
"message": "One or more variations of this test failed. Details can be found under each [variant#].",
163+
"test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
164+
"task_id": 2
165+
},
173166
{
174167
"name": "ExampleAllFailOther > hello [variation #1]",
175168
"status": "fail",
@@ -197,6 +190,13 @@
197190
"message": "AssertionError: 'Goodbye!' != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.",
198191
"test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
199192
"task_id": 2
193+
},
194+
{
195+
"name": "ExampleAllFailOther > hello~",
196+
"status": "fail",
197+
"message": "One or more variations of this test failed. Details can be found under each [variant#].",
198+
"test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n failure_msg=f'Expected: {result} but got something else instead.'\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result, msg=failure_msg)",
199+
"task_id": 2
200200
}
201201
]
202202
}

‎test/example-config-multiple-files-subtests-and-tasks/results.json‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,56 +3,56 @@
33
"status": "pass",
44
"tests": [
55
{
6-
"name": "ExampleFirst > abc",
6+
"name": "ExampleFirst > abc~",
77
"status": "pass",
88
"test_code": "self.assertEqual(hello(13), (\"Hello, World!\", 13))",
99
"task_id": 1,
1010
"output": "User output is captured! 13"
1111
},
1212
{
13-
"name": "ExampleFirst > hello",
13+
"name": "ExampleFirst > hello~",
1414
"status": "pass",
1515
"test_code": "self.assertEqual(hello('Hi'), (\"Hello, World!\", 'Hi'))",
1616
"task_id": 1,
1717
"output": "User output is captured! Hi"
1818
},
1919
{
20-
"name": "ExampleFirstOther > dummy",
20+
"name": "ExampleFirstOther > dummy~",
2121
"status": "pass",
2222
"test_code": "self.assertEqual(hello('Banana'), (\"Hello, World!\", \"Banana\"))",
2323
"task_id": 2,
2424
"output": "User output is captured! Banana"
2525
},
2626
{
27-
"name": "ExampleFirstOther > hello",
27+
"name": "ExampleFirstOther > hello~",
2828
"status": "pass",
2929
"test_code": "self.assertEqual(hello(42), (\"Hello, World!\", 42))",
3030
"task_id": 2,
3131
"output": "User output is captured! 42"
3232
},
3333
{
34-
"name": "ExampleSecond > abc",
34+
"name": "ExampleSecond > abc~",
3535
"status": "pass",
3636
"test_code": "input_data = ['carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')",
3737
"task_id": 3,
3838
"output": "User output is captured! carrot\nUser output is captured! cucumber\nUser output is captured! grass\nUser output is captured! tree"
3939
},
4040
{
41-
"name": "ExampleSecond > hello",
41+
"name": "ExampleSecond > hello~",
4242
"status": "pass",
4343
"test_code": "input_data = [1, 2, 5]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')",
4444
"task_id": 3,
4545
"output": "User output is captured! 1\nUser output is captured! 2\nUser output is captured! 5"
4646
},
4747
{
48-
"name": "ExampleSecondOther > dummy",
48+
"name": "ExampleSecondOther > dummy~",
4949
"status": "pass",
5050
"test_code": "input_data = ['frog', 'fish', 'coconut']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')",
5151
"task_id": 4,
5252
"output": "User output is captured! frog\nUser output is captured! fish\nUser output is captured! coconut"
5353
},
5454
{
55-
"name": "ExampleSecondOther > hello",
55+
"name": "ExampleSecondOther > hello~",
5656
"status": "pass",
5757
"test_code": "input_data = [23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n with self.subTest(f\"variation #{variant}\", param=param, result=result):\n self.assertEqual(hello(param), result,\n msg=f'Expected: {result} but got something else instead.')",
5858
"task_id": 4,

‎test/example-has-stdout-and-tasks/results.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,39 +3,39 @@
33
"status": "fail",
44
"tests": [
55
{
6-
"name": "ExampleHasStdoutAndTasks > hello",
6+
"name": "ExampleHasStdoutAndTasks > hello~",
77
"status": "fail",
88
"message": "AssertionError: None != 'Hello, World!'",
99
"test_code": "self.assertEqual(hello(), \"Hello, World!\")",
1010
"task_id": 1,
1111
"output": "Hello, World!"
1212
},
1313
{
14-
"name": "ExampleHasStdoutAndTasks > abc",
14+
"name": "ExampleHasStdoutAndTasks > abc~",
1515
"status": "fail",
1616
"message": "AssertionError: None != 'Hello, World!'",
1717
"test_code": "self.assertEqual(hello(), \"Hello, World!\")",
1818
"task_id": 2,
1919
"output": "Hello, World!"
2020
},
2121
{
22-
"name": "ExampleHasStdoutAndTasks > truncation",
22+
"name": "ExampleHasStdoutAndTasks > truncation~",
2323
"status": "fail",
2424
"message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!",
2525
"test_code": "self.assertEqual(must_truncate(), \"Hello, World!\")",
2626
"task_id": 3,
2727
"output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]"
2828
},
2929
{
30-
"name": "ExampleHasStdoutAndTasksOther > dummy",
30+
"name": "ExampleHasStdoutAndTasksOther > dummy~",
3131
"status": "fail",
3232
"message": "AssertionError: None != 'Hello, World!'",
3333
"test_code": "self.assertEqual(hello(), \"Hello, World!\")",
3434
"task_id": 4,
3535
"output": "Hello, World!"
3636
},
3737
{
38-
"name": "ExampleHasStdoutAndTasksOther > hello",
38+
"name": "ExampleHasStdoutAndTasksOther > hello~",
3939
"status": "fail",
4040
"message": "AssertionError: None != 'Hello, World!'",
4141
"test_code": "self.assertEqual(hello(), \"Hello, World!\")",

0 commit comments

Comments
 (0)