Better side-by-side (#306)

* Better side-by-side

* fix command lien

* last changes

* small changes

Author: xadupre

CHANGELOGS.rst

@@ -4,7 +4,7 @@ Change Logs
 0.8.3
 +++++
 
-* :pr:`304`: improves side-by-side comparison
+* :pr:`304`, :pr:`306`: improves side-by-side comparison
 
 0.8.2
 +++++

_unittests/ut_torch_onnx/test_sbs.py

@@ -10,7 +10,7 @@
 )
 from onnx_diagnostic.reference import ExtendedReferenceEvaluator, OnnxruntimeEvaluator
 from onnx_diagnostic.torch_export_patches.patch_inputs import use_dyn_not_str
-from onnx_diagnostic.torch_onnx.sbs import run_aligned, post_process_run_aligned_obs
+from onnx_diagnostic.torch_onnx.sbs import run_aligned, RunAlignedRecord
 from onnx_diagnostic.export.api import to_onnx
 
 
@@ -21,6 +21,24 @@ def setUpClass(cls):
 
         cls.torch = torch
 
+    def test_run_aligned_record(self):
+        r = RunAlignedRecord(
+            ep_id_node=-1,
+            onnx_id_node=-1,
+            ep_name="A",
+            onnx_name="B",
+            ep_target="C",
+            onnx_op_type="D",
+            shape_type="E",
+            err_abs=0.1,
+            err_rel=0.2,
+            err_dev=0.3,
+            err_nan=0.4,
+        )
+        sr = str(r)
+        self.assertIn("RunAlignedRecord(", sr)
+        self.assertIn("shape_type='E'", sr)
+
     @hide_stdout()
     @unittest.skipIf(to_onnx is None, "to_onnx not installed")
     @ignore_errors(OSError)  # connectivity issues
@@ -48,7 +66,7 @@ def forward(self, x):
                 run_cls=ExtendedReferenceEvaluator,
                 atol=1e-5,
                 rtol=1e-5,
-                verbose=1,
+                verbose=10,
             ),
         )
         self.assertEqual(len(results), 7)
@@ -83,7 +101,7 @@ def forward(self, x):
                 run_cls=ExtendedReferenceEvaluator,
                 atol=1e-5,
                 rtol=1e-5,
-                verbose=1,
+                verbose=10,
             ),
         )
         self.assertEqual(len(results), 6)
@@ -115,7 +133,7 @@ def forward(self, x):
                 run_cls=ExtendedReferenceEvaluator,
                 atol=1e-5,
                 rtol=1e-5,
-                verbose=1,
+                verbose=10,
             ),
         )
         self.assertEqual(len(results), 6)
@@ -182,7 +200,6 @@ def forward(self, x):
             ),
         )
         self.assertEqual(len(results), 8)
-        self.clean_dump()
 
     @hide_stdout()
     @ignore_warnings((DeprecationWarning, FutureWarning, UserWarning))
@@ -285,7 +302,10 @@ def forward(self, x):
             ),
         )
         self.assertEqual(len(results), 14)
-        self.assertEqual([r[-1].get("dev", 0) for r in results], [0] * 14)
+        self.assertEqual(
+            [r.err_dev for r in results],
+            [None, None, None, None, None, None, None, None, 0, 0, 0, 0, 0, 0],
+        )
 
     @hide_stdout()
     @ignore_warnings((DeprecationWarning, FutureWarning, UserWarning))
@@ -323,25 +343,32 @@ def forward(self, x):
                 use_tensor=True,
             ),
         )
-        df = pandas.DataFrame(list(map(post_process_run_aligned_obs, results)))
+        df = pandas.DataFrame(list(results))
         df.to_excel(self.get_dump_file("test_sbs_model_with_weights_custom.xlsx"))
         self.assertEqual(
             [
                 "ep_id_node",
                 "ep_name",
                 "ep_target",
+                "ep_time_run",
                 "err_abs",
                 "err_dev",
+                "err_nan",
                 "err_rel",
                 "onnx_id_node",
+                "onnx_id_output",
                 "onnx_name",
                 "onnx_op_type",
+                "onnx_time_run",
                 "shape_type",
             ],
             sorted(df.columns),
         )
         self.assertEqual(len(results), 12)
-        self.assertEqual([r[-1].get("dev", 0) for r in results], [0] * 12)
+        self.assertEqual(
+            [r.err_dev for r in results],
+            [None, None, None, None, None, None, None, None, None, 0, 0, 0],
+        )
         self.assertEqual(
             [-1.0, -1.0, -1.0, -1.0, -10.0, -10.0, -10.0, -10.0, -1.0, 0.0, 1.0, 2.0],
             df["onnx_id_node"].fillna(-10).tolist(),
@@ -384,25 +411,32 @@ def forward(self, x):
                 use_tensor=True,
             ),
         )
-        df = pandas.DataFrame(list(map(post_process_run_aligned_obs, results)))
+        df = pandas.DataFrame(list(results))
         df.to_excel(self.get_dump_file("test_sbs_model_with_weights_dynamo.xlsx"))
         self.assertEqual(
             [
                 "ep_id_node",
                 "ep_name",
                 "ep_target",
+                "ep_time_run",
                 "err_abs",
                 "err_dev",
+                "err_nan",
                 "err_rel",
                 "onnx_id_node",
+                "onnx_id_output",
                 "onnx_name",
                 "onnx_op_type",
+                "onnx_time_run",
                 "shape_type",
             ],
             sorted(df.columns),
         )
         self.assertEqual(len(results), 12)
-        self.assertEqual([r[-1].get("dev", 0) for r in results], [0] * 12)
+        self.assertEqual(
+            [r.err_dev for r in results],
+            [None, None, None, None, None, None, None, None, None, 0, 0, 0],
+        )
         self.assertEqual(
             [-1.0, -1.0, -1.0, -1.0, -10.0, -10.0, -10.0, -10.0, -1.0, 0.0, 1.0, 2.0],
             df["onnx_id_node"].fillna(-10).tolist(),

onnx_diagnostic/_command_lines_parser.py

@@ -1114,10 +1114,20 @@ def get_parser_sbs() -> ArgumentParser:
             the exported onnx model. It assumes some names are common.
             The execution of the exported program and the onnx model
             are done in parallel. The device is the one used to store the
-            model and the inputs.s
+            model and the inputs.
+            Where do discrepancies start? This function tries to answer that question.
+            """
+        ),
+        epilog=textwrap.dedent(
+            """
+            The command line expects the following files to be saved with
+            the following function. inputs is a dictionary of the input of the model.
+
+            - torch.export.save(ep: torch.export.ExportedProgram)
+            - torch.save(**inputs)
+            - onnx.save(...)
             """
         ),
-        epilog="Where do discrepancies start? This function tries to answer that question.",
     )
     parser.add_argument(
         "-i",
@@ -1169,9 +1179,9 @@ def get_parser_sbs() -> ArgumentParser:
     parser.add_argument(
         "-r",
         "--ratio",
-        default=5,
+        default=100,
         required=False,
-        help="Saves the result in an excel file every <ratio> node.",
+        help="Saves the result in an excel file every <ratio> nodes.",
     )
     return parser
 
@@ -1180,7 +1190,7 @@ def _cmd_sbs(argv: List[Any]):
     import pandas
     import torch
     from .helpers import string_type
-    from .torch_onnx.sbs import run_aligned, post_process_run_aligned_obs
+    from .torch_onnx.sbs import run_aligned
     from .reference import OnnxruntimeEvaluator
 
     parser = get_parser_sbs()
@@ -1241,13 +1251,20 @@ def _size(name):
         use_tensor=True,
         exc=False,
     ):
-        pobs = post_process_run_aligned_obs(obs)
-        data.append(pobs)
-        if "initializer" not in pobs and "placeholder" not in pobs and len(data) % ratio == 0:
-            df = pandas.DataFrame(data)
+        data.append(obs)
+        if (
+            obs.onnx_op_type != "initializer"
+            and obs.ep_target != "placeholder"
+            and len(data) % ratio == 0
+        ):
+            df = pandas.DataFrame(data).apply(
+                lambda col: col.fillna("") if col.dtype == "object" else col
+            )
             df.to_excel(args.output)
     print(f"-- final saves into {args.output!r}")
-    df = pandas.DataFrame(data)
+    df = pandas.DataFrame(data).apply(
+        lambda col: col.fillna("") if col.dtype == "object" else col
+    )
     df.to_excel(args.output)
     print("-- done")