improves side by side (#304)

* improves side by side

* mypy

* changes

* mypy

* fixes

* fix ut

* doc

* sbs

* doc

* last

* fix

* fix

* fix sbs

* sbs

* fix sbs

* fix

* fix

* mypy"
git push

"

* onx fix

* fix mypy

* fix missing domain

* mypy

Author: xadupre

.gitignore

@@ -14,6 +14,8 @@
 *.weight
 *.nsys-rep
 *.pkl
+*.pt
+*.pt2
 *.xlsx
 *.sarif
 *.sqlitest

CHANGELOGS.rst

@@ -1,6 +1,11 @@
 Change Logs
 ===========
 
+0.8.3
++++++
+
+* :pr:`304`: improves side-by-side comparison
+
 0.8.2
 +++++
 

_doc/index.rst

@@ -239,8 +239,8 @@ The function replaces dynamic dimensions defined as strings by
 Older versions
 ==============
 
+* `0.8.3 <../v0.8.3/index.html>`_
 * `0.8.2 <../v0.8.2/index.html>`_
-* `0.8.1 <../v0.8.1/index.html>`_
 * `0.7.16 <../v0.7.16/index.html>`_
 * `0.6.3 <../v0.6.3/index.html>`_
 * `0.5.0 <../v0.5.0/index.html>`_

_unittests/ut_export/test_api.py

@@ -110,6 +110,8 @@ def test_tiny_llm_to_onnx(self):
                 diff = max_diff(expected, got)
                 assert diff["abs"] <= 1e-5, f"diff={diff}"
 
+        self.clean_dump()
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

_unittests/ut_helpers/test_bench_run.py

@@ -109,35 +109,51 @@ def test_make_configs_replace(self):
     def test_max_diff(self):
         self.assertEqual(
             max_diff(torch.Tensor([1, 2]), torch.Tensor([1, 2])),
-            {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 2.0, "dnan": 0.0, "argm": (0,)},
+            {
+                "abs": 0.0,
+                "rel": 0.0,
+                "sum": 0.0,
+                "n": 2.0,
+                "dnan": 0.0,
+                "argm": (0,),
+                "dev": 0,
+            },
         )
         self.assertEqual(
             max_diff(
                 (torch.Tensor([1, 2]),),
                 (torch.Tensor([1, 2])),
             ),
-            {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 2.0, "dnan": 0.0, "argm": (0,)},
+            {
+                "abs": 0.0,
+                "rel": 0.0,
+                "sum": 0.0,
+                "n": 2.0,
+                "dnan": 0.0,
+                "argm": (0,),
+                "dev": 0,
+            },
         )
         self.assertEqual(
             max_diff(
                 (torch.Tensor([1, 2]), (torch.Tensor([1, 2]),)),
                 (torch.Tensor([1, 2]), (torch.Tensor([1, 2]),)),
             ),
-            {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 4.0, "dnan": 0.0},
+            {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 4.0, "dnan": 0.0, "dev": 0},
         )
         self.assertEqual(
             max_diff(
                 {"a": torch.Tensor([1, 2])},
                 {"a": torch.Tensor([1, 2])},
             ),
-            {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 2.0, "dnan": 0.0},
+            {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 2.0, "dnan": 0.0, "dev": 0},
         )
         self.assertEqual(
             max_diff(
                 {"a": torch.Tensor([1, 2])},
                 [torch.Tensor([1, 2])],
             ),
-            {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 2.0, "dnan": 0.0},
+            {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 2.0, "dnan": 0.0, "dev": 0},
         )
         self.assertEqual(
             max_diff(
@@ -150,6 +166,7 @@ def test_max_diff(self):
                 "n": 2.0,
                 "rel": 0.9999999997999001,
                 "sum": 9999999998.0,
+                "dev": 0,
             },
         )
 
@@ -164,7 +181,9 @@ def test_max_diff_dynamic_cache(self):
             flatten=True,
             verbose=10,
         )
-        self.assertEqual(md, {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 10.0, "dnan": 0})
+        self.assertEqual(
+            md, {"abs": 0.0, "rel": 0.0, "sum": 0.0, "n": 10.0, "dnan": 0, "dev": 0}
+        )
 
 
 if __name__ == "__main__":

_unittests/ut_helpers/test_helper.py

@@ -11,6 +11,7 @@
     hide_stdout,
     requires_onnx,
     requires_transformers,
+    requires_cuda,
 )
 from onnx_diagnostic.helpers.helper import (
     string_type,
@@ -199,6 +200,31 @@ def test_flatten(self):
         d = string_diff(diff)
         self.assertIsInstance(d, str)
 
+    @hide_stdout()
+    def test_maxdiff_device(self):
+        inputs = (torch.arange(2), torch.cos(torch.arange(3)))
+        diff = max_diff(inputs, inputs, flatten=True, verbose=10)
+        self.assertEqual(diff["abs"], 0)
+        self.assertEqual(diff["dev"], 0)
+
+    @hide_stdout()
+    @requires_cuda()
+    def test_maxdiff_device_cuda(self):
+        diff = max_diff(torch.ones((2,)).cuda(), torch.ones((2,)), verbose=10)
+        self.assertEqual(diff["dev"], 1)
+        inputs = (torch.arange(2), torch.cos(torch.arange(3)))
+        inputs2 = (inputs[0].cuda(), inputs[1].cuda())
+        diff = max_diff(inputs, inputs2, verbose=10)
+        self.assertEqual(diff["abs"], 0)
+        self.assertEqual(diff["dev"], 2)
+        inputs2 = (inputs[0], inputs[1].cuda())
+        diff = max_diff(inputs, inputs2, verbose=10)
+        self.assertEqual(diff["abs"], 0)
+        self.assertEqual(diff["dev"], 1)
+        diff = max_diff(inputs2, inputs2, verbose=10)
+        self.assertEqual(diff["abs"], 0)
+        self.assertEqual(diff["dev"], 0)
+
     def test_flatten_cache(self):
         cache = make_dynamic_cache([(torch.ones((5, 6, 5, 6)), torch.ones((5, 6, 5, 6)) + 2)])
         flat = flatten_object(cache, drop_keys=True)

_unittests/ut_helpers/test_log_helper.py

@@ -189,6 +189,7 @@ def test_cube_logs_excel(self):
             verbose=1,
         )
         self.assertExists(output)
+        self.clean_dump()
 
     @hide_stdout()
     def test_enumerate_csv_files(self):
@@ -210,6 +211,7 @@ def test_enumerate_csv_files(self):
         cube.load(verbose=1)
         self.assertEqual((3, 11), cube.shape)
         self.assertIn("RAWFILENAME", cube.data.columns)
+        self.clean_dump()
 
     def test_cube_logs_performance1(self):
         output = self.get_dump_file("test_cube_logs_performance1.xlsx")
@@ -235,6 +237,7 @@ def test_cube_logs_performance1(self):
             ],
         )
         self.assertExists(output)
+        self.clean_dump()
 
     def test_cube_logs_performance2(self):
         output = self.get_dump_file("test_cube_logs_performance2.xlsx")
@@ -470,6 +473,7 @@ def test_historical_cube_time_mask(self):
         )
         cube = CubeLogs(df, keys=["^m_*", "exporter"], time="date").load()
         cube.to_excel(output, views=["time_p"], time_mask=True, verbose=1)
+        self.clean_dump()
 
     def test_cube_sbs_no_time(self):
         df = pandas.DataFrame(
@@ -532,6 +536,7 @@ def test_cube_sbs_no_time(self):
             verbose=0,
             sbs=dict(CFA=dict(exporter="E1", opt="O"), CFB=dict(exporter="E2", opt="O")),
         )
+        self.clean_dump()
 
     def test_cube_sbs_with_time(self):
         df = pandas.DataFrame(

_unittests/ut_helpers/test_ort_session.py

@@ -310,6 +310,7 @@ def test_profiling(self):
         got = wrap.run(None, feeds)
         self.assertIsInstance(got[0], torch.Tensor)
         self.assertEqualArray(expected[0], got[0])
+        self.clean_dump()
 
 
 if __name__ == "__main__":

_unittests/ut_helpers/test_torch_helper.py

@@ -151,6 +151,7 @@ def forward(self, x, y):
         self.assertEqualAny(restored["main", 1, "I"], (inputs, {}))
         self.assertEqualAny(restored["main", 0, "O"], res1)
         self.assertEqualAny(restored["main", 0, "O"], res2)
+        self.clean_dump()
 
     @hide_stdout()
     def test_steal_forward_dump_file_steal_append(self):
@@ -181,6 +182,7 @@ def forward(self, x, y):
             {("", 1, "I"), ("", 1, "O"), "sx", ("", 0, "O"), "sx_1", ("", 0, "I")},
             set(restored),
         )
+        self.clean_dump()
 
     @hide_stdout()
     def test_steal_forward_dump_file_steal_append_drop(self):
@@ -214,6 +216,7 @@ def forward(self, x, y):
         first = restored[("", 0, "I")]
         _a, kws = first
         self.assertNotIn("x", kws)
+        self.clean_dump()
 
     @hide_stdout()
     def test_steal_forward_submodules(self):
@@ -257,6 +260,7 @@ def forward(self, x, y):
             ),
             len(sorted(restored)),
         )
+        self.clean_dump()
 
     def test_replace_string_by_dynamic(self):
         example = {

_unittests/ut_reference/test_onnxruntime_evaluator.py

@@ -1,10 +1,11 @@
 import unittest
+from typing import Optional
 import numpy as np
 import onnx
 import onnx.helper as oh
 import torch
 import onnxruntime
-from onnx_diagnostic.ext_test_case import ExtTestCase, hide_stdout
+from onnx_diagnostic.ext_test_case import ExtTestCase, hide_stdout, ignore_warnings
 from onnx_diagnostic.helpers.onnx_helper import from_array_extended
 from onnx_diagnostic.reference import (
     OnnxruntimeEvaluator,
@@ -22,6 +23,14 @@
 
 
 class TestOnnxruntimeEvaluator(ExtTestCase):
+    def _range(self, *shape, bias: Optional[float] = None):
+        n = np.prod(shape)
+        x = np.arange(n).astype(np.float32) / n
+        if bias:
+            x = x + bias
+        return x.reshape(tuple(shape)).astype(np.float32)
+
+    @ignore_warnings(FutureWarning)
     def test_ort_eval_scan_cdist_add(self):
 
         def dist(unused: torch.Tensor, x: torch.Tensor, samex: torch.Tensor):
@@ -69,6 +78,7 @@ def forward(self, x):
         got = orte.run(None, {name: x.numpy()})[0]
         self.assertEqualArray(expected, got)
 
+    @ignore_warnings((UserWarning, FutureWarning))
     def test_ort_eval_cond(self):
         import torch
 
@@ -180,6 +190,7 @@ def test_constant_bool_input(self):
         self.assertEqual(got.dtype, torch.bool)
         self.assertEqual(got[0], True)
 
+    @hide_stdout()
     def test_ort_eval_loop(self):
         model = torch.nn.EmbeddingBag(num_embeddings=49157, embedding_dim=32, mode="sum")
         a = torch.tensor([[39906, 39906]]).long()
@@ -226,6 +237,28 @@ def test_report_results_comparison_ort(self):
         self.assertLess(d[(0, "nx"), "r_cos"], 1e-6)
         self.assertLess(d[(2, "u"), "r_exp"], 1e-6)
 
+    @hide_stdout()
+    def test_skip_layer_normalization(self):
+        node = oh.make_node(
+            "SkipLayerNormalization",
+            ["x", "skip", "beta", "gamma", "bias"],
+            ["Z"],
+            epsilon=1.0e-5,
+            domain="com.microsoft",
+        )
+        feeds = dict(
+            x=self._range(2, 3, 8),
+            skip=self._range(2, 3, 8, bias=3),
+            beta=self._range(8, bias=1),
+            gamma=self._range(8, bias=2),
+            bias=self._range(8, bias=0.1),
+        )
+        ref = ExtendedReferenceEvaluator(node)
+        expected = ref.run(None, feeds)
+        rt = OnnxruntimeEvaluator(node, verbose=10, opsets={"": 22})
+        got = rt.run(None, feeds)
+        self.assertEqualAny(expected, got, atol=1e-4)
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)