Generate the testing report Use the report-summary module to create a report for the frontend with clear structure.

- Add the tolerance values for each test case instead of using fixed value for all tests.

Author: AhmedBasem20

.github/workflows/website.yml

@@ -48,15 +48,16 @@ jobs:
 
       - name: Run the test that generates the plots report.
         run: |
-          pytest tests/IVIMmodels/unit_tests/test_ivim_fit.py
+          pytest tests/IVIMmodels/unit_tests/test_ivim_fit.py --json-report
+          python utilities/report-summary.py .report.json report-summary.json
 
       - name: 'Filter and compress results file.'
         run: python utilities/reduce_output_size.py test_output.csv test_output.csv.gz
 
       - name: move data to the dashboard folder
         run: |
           mv test_output.csv.gz website/dashboard
-          mv test_results_report.json website/dashboard
+          mv report-summary.json website/dashboard
 
 
       - name: Build documentation

tests/IVIMmodels/unit_tests/test_ivim_fit.py

@@ -70,7 +70,7 @@ def data_ivim_fit_saved():
 
 
 @pytest.mark.parametrize("name, bvals, data, algorithm, xfail, kwargs, tolerances", data_ivim_fit_saved())
-def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances, request, test_results):
+def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances, request, record_property):
     if xfail["xfail"]:
         mark = pytest.mark.xfail(reason="xfail", strict=xfail["strict"])
         request.node.add_marker(mark)
@@ -90,12 +90,12 @@ def to_list_if_needed(value):
         "f": to_list_if_needed(data['f']),
         "Dp": to_list_if_needed(data['Dp']),
         "D": to_list_if_needed(data['D']),
-        "status": "PASSED"
+        "rtol": tolerances["rtol"],
+        "atol": tolerances["atol"]
     }
-    if xfail["xfail"]:
-        test_result['status'] = "XFAILED"
 
-    test_results.append(test_result)
+
+    record_property('test_data', test_result)
 
     npt.assert_allclose(data['f'], f_fit, rtol=tolerances["rtol"]["f"], atol=tolerances["atol"]["f"])
     npt.assert_allclose(data['D'], D_fit, rtol=tolerances["rtol"]["D"], atol=tolerances["atol"]["D"])

utilities/report-summary.py

@@ -0,0 +1,26 @@
+import pathlib
+import json
+import sys
+
+def summarize_test_report(input_file:str, output_file:str):
+    file = pathlib.Path(__file__)
+    report_path = file.with_name(input_file)
+    with report_path.open() as f:
+        report_info = json.load(f)
+    summary = []
+    for test_case in report_info['tests']:
+        values = test_case['user_properties'][0]['test_data']
+        values['status'] = test_case['outcome']
+        summary.append(values)
+
+    with open(output_file, 'w') as f:
+        json.dump(summary, f, indent=4)
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print("Usage: python report-summary.py <input_file> <output_file>")
+        sys.exit(1)
+    
+    input_file = sys.argv[1]
+    output_file = sys.argv[2]
+    summarize_test_report(input_file, output_file)

website/dashboard/test_plots.js

@@ -1,18 +1,14 @@
 document.addEventListener('DOMContentLoaded', function() {
-    fetch('test_results_report.json')
+    fetch('report-summary.json')
         .then(response => response.json())
         .then(data => {
-            var rtol = data['rtol']
-
-            var atol = data['atol']
-
-            function createPlot(container, parameter, rtol_value, atol_value) {
-                var reference_values = data['results'].map(d => d[parameter]); // Assuming fit values are the reference
-                var fit_values = data['results'].map(d => d[parameter + '_fit']);
+            function createPlot(container, parameter) {
+                var reference_values = data.map(d => d[parameter]);
+                var fit_values = data.map(d => d[parameter + '_fit']);
                 var errors = fit_values.map((d, i) => d - reference_values[i]);
 
-                var tolerance = reference_values.map(d => atol_value + rtol_value * d);
-                var negative_tolerance = reference_values.map(d => -(atol_value + rtol_value * d));
+                var tolerance = reference_values.map((d, i) => data[i]['atol'][parameter] + data[i]['rtol'][parameter] * d);
+                var negative_tolerance = reference_values.map((d, i) => -(data[i]['atol'][parameter] + data[i]['rtol'][parameter] * d));
 
                 var scatter_trace = {
                     x: reference_values,
@@ -51,8 +47,8 @@ document.addEventListener('DOMContentLoaded', function() {
                 Plotly.newPlot(container, plot_data, layout);
             }
 
-            createPlot('plot_f_fit', 'f', rtol.f, atol.f);
-            createPlot('plot_Dp_fit', 'Dp', rtol.Dp, atol.Dp);
-            createPlot('plot_D_fit', 'D', rtol.D, atol.D);
+            createPlot('plot_f_fit', 'f');
+            createPlot('plot_Dp_fit', 'Dp');
+            createPlot('plot_D_fit', 'D');
         });
 });