CHANGED: Expand Optimized Controls performance testing (#1905)

jfreire-unity · web-flow · commit 335be330d5a5 · 2024-04-18T18:37:14.000+02:00
* Update existing performance tests

It will make the tests run for more cases to see the impact of each internal feature flag.

* Add tests for more use cases

Adds a gamepad and mouse tests. This showcases that the optimizations won't improve performance for all use cases.

* Add test which only calls InputSystem.Update()

Evaluates cases when there are no control changes or reads from the user to assess if there's any impact by the optimizations.

* Add test that shows degraded performance.

When using ReadCacheValue for devices that have state updates every frame, there's a performance cost.

* Add missing comments to some tests
diff --git a/Assets/Tests/InputSystem/CorePerformanceTests.cs b/Assets/Tests/InputSystem/CorePerformanceTests.cs
@@ -546,24 +546,41 @@ public void TODO_CanSaveAndRestoreSystemInLessThan10Milliseconds() // Currently
 
     #endif
 
-    internal enum OptimizedControlsTest
+    internal enum OptimizationTestType
     {
+        NoOptimization,
         OptimizedControls,
-        NormalControls
+        ReadValueCaching,
+        OptimizedControlsAndReadValueCaching
     }
 
-    [Test, Performance]
-    [Category("Performance")]
-    [TestCase(OptimizedControlsTest.OptimizedControls)]
-    [TestCase(OptimizedControlsTest.NormalControls)]
-    public void Performance_OptimizedControls_ReadingMousePosition100kTimes(OptimizedControlsTest testSetup)
+    public void SetInternalFeatureFlagsFromTestType(OptimizationTestType testType)
     {
-        var useOptimizedControls = testSetup == OptimizedControlsTest.OptimizedControls;
+        var useOptimizedControls = testType == OptimizationTestType.OptimizedControls
+            || testType == OptimizationTestType.OptimizedControlsAndReadValueCaching;
+        var useReadValueCaching = testType == OptimizationTestType.ReadValueCaching
+            || testType == OptimizationTestType.OptimizedControlsAndReadValueCaching;
+
         InputSystem.settings.SetInternalFeatureFlag(InputFeatureNames.kUseOptimizedControls, useOptimizedControls);
-        InputSystem.settings.SetInternalFeatureFlag(InputFeatureNames.kUseReadValueCaching, useOptimizedControls);
+        InputSystem.settings.SetInternalFeatureFlag(InputFeatureNames.kUseReadValueCaching, useReadValueCaching);
         InputSystem.settings.SetInternalFeatureFlag(InputFeatureNames.kParanoidReadValueCachingChecks, false);
+    }
+
+    [Test, Performance]
+    [Category("Performance")]
+    [TestCase(OptimizationTestType.NoOptimization)]
+    [TestCase(OptimizationTestType.OptimizedControls)]
+    [TestCase(OptimizationTestType.ReadValueCaching)]
+    [TestCase(OptimizationTestType.OptimizedControlsAndReadValueCaching)]
+    // Isolated tests for reading from Mouse device to evaluate the performance of the optimizations.
+    // Does not take into account the performance of the InputSystem.Update() call.
+    public void Performance_OptimizedControls_ReadingMousePosition100kTimes(OptimizationTestType testType)
+    {
+        SetInternalFeatureFlagsFromTestType(testType);
 
         var mouse = InputSystem.AddDevice<Mouse>();
+        var useOptimizedControls = testType == OptimizationTestType.OptimizedControls
+            || testType == OptimizationTestType.OptimizedControlsAndReadValueCaching;
         Assert.That(mouse.position.x.optimizedControlDataType, Is.EqualTo(useOptimizedControls ? InputStateBlock.FormatFloat : InputStateBlock.FormatInvalid));
         Assert.That(mouse.position.y.optimizedControlDataType, Is.EqualTo(useOptimizedControls ? InputStateBlock.FormatFloat : InputStateBlock.FormatInvalid));
         Assert.That(mouse.position.optimizedControlDataType, Is.EqualTo(useOptimizedControls ? InputStateBlock.FormatVector2 : InputStateBlock.FormatInvalid));
@@ -579,17 +596,188 @@ public void Performance_OptimizedControls_ReadingMousePosition100kTimes(Optimize
             .Run();
     }
 
+    [Test, Performance]
+    [Category("Performance")]
+    [TestCase(OptimizationTestType.NoOptimization)]
+    [TestCase(OptimizationTestType.OptimizedControls)]
+    [TestCase(OptimizationTestType.ReadValueCaching)]
+    [TestCase(OptimizationTestType.OptimizedControlsAndReadValueCaching)]
+    // Currently these tests shows that all the optimizations have a performance cost when reading from a Mouse device.
+    // OptimizedControls option is slower because of an extra check that is only done in Editor and Development Builds.
+    // ReadValueCaching option is slower because Mouse state (FastMouse) is changed every update, which means cached
+    // values are always stale. And currently there is a cost when caching the value.
+    public void Performance_OptimizedControls_ReadAndUpdateMousePosition1kTimes(OptimizationTestType testType)
+    {
+        SetInternalFeatureFlagsFromTestType(testType);
+
+        var mouse = InputSystem.AddDevice<Mouse>();
+        InputSystem.Update();
+
+        var useOptimizedControls = testType == OptimizationTestType.OptimizedControls
+            || testType == OptimizationTestType.OptimizedControlsAndReadValueCaching;
+        Assert.That(mouse.position.x.optimizedControlDataType, Is.EqualTo(useOptimizedControls ? InputStateBlock.FormatFloat : InputStateBlock.FormatInvalid));
+        Assert.That(mouse.position.y.optimizedControlDataType, Is.EqualTo(useOptimizedControls ? InputStateBlock.FormatFloat : InputStateBlock.FormatInvalid));
+        Assert.That(mouse.position.optimizedControlDataType, Is.EqualTo(useOptimizedControls ? InputStateBlock.FormatVector2 : InputStateBlock.FormatInvalid));
+
+        Measure.Method(() =>
+        {
+            var pos = new Vector2();
+            for (var i = 0; i < 1000; ++i)
+            {
+                pos += mouse.position.ReadValue();
+                InputSystem.Update();
+
+                if (i % 100 == 0)
+                {
+                    // Make sure there's a new different value every 100 frames.
+                    InputSystem.QueueStateEvent(mouse, new MouseState { position = new Vector2(i + 1, i + 2) });
+                }
+            }
+        })
+            .MeasurementCount(100)
+            .WarmupCount(5)
+            .Run();
+    }
+
+    [Test, Performance]
+    [Category("Performance")]
+    [TestCase(OptimizationTestType.NoOptimization)]
+    [TestCase(OptimizationTestType.ReadValueCaching)]
+    // These tests shows a use case where ReadValueCaching optimization will perform better than without any
+    // optimization.
+    // It shows that there's a performance improvement when the control values being read are not changing every frame.
+    public void Performance_OptimizedControls_ReadAndUpdateGamepad1kTimes(OptimizationTestType testType)
+    {
+        SetInternalFeatureFlagsFromTestType(testType);
+
+        var gamepad = InputSystem.AddDevice<Gamepad>();
+
+        InputSystem.Update();
+
+        Measure.Method(() =>
+        {
+            var pos = new Vector2();
+            InputSystem.QueueStateEvent(gamepad, new GamepadState { leftStick = new Vector2(0.3f, 0.1f) });
+            InputSystem.Update();
+
+            pos = gamepad.leftStick.value;
+            Assert.That(gamepad.leftStick.m_CachedValueIsStale, Is.False);
+
+            for (var i = 0; i < 1000; ++i)
+            {
+                InputSystem.Update();
+                pos = gamepad.leftStick.value;
+                Assert.That(gamepad.leftStick.m_CachedValueIsStale, Is.False);
+
+                if (i % 100 == 0)
+                {
+                    // Make sure there's a new different value every 100 frames to mark the cached value as stale.
+                    InputSystem.QueueStateEvent(gamepad, new GamepadState { leftStick = new Vector2(i / 1000f, i / 1000f) });
+                    InputSystem.Update();
+                }
+            }
+        })
+            .MeasurementCount(100)
+            .WarmupCount(10)
+            .Run();
+    }
+
+    [Test, Performance]
+    [Category("Performance")]
+    [TestCase(OptimizationTestType.NoOptimization)]
+    [TestCase(OptimizationTestType.ReadValueCaching)]
+    // This shows a use case where ReadValueCaching optimization will perform worse when controls have stale cached
+    // values every frame. Meaning, when control values change in every frame.
+    public void Performance_OptimizedControls_ReadAndUpdateGamepadNewValuesEveryFrame1kTimes(OptimizationTestType testType)
+    {
+        SetInternalFeatureFlagsFromTestType(testType);
+
+        var gamepad = InputSystem.AddDevice<Gamepad>();
+
+        InputSystem.Update();
+
+        Measure.Method(() =>
+        {
+            var pos = new Vector2();
+            InputSystem.QueueStateEvent(gamepad, new GamepadState { leftStick = new Vector2(0.1f, 0.1f) });
+            InputSystem.Update();
+
+            gamepad.leftStick.ReadValue();
+            Assert.That(gamepad.leftStick.m_CachedValueIsStale, Is.False);
+
+            for (var i = 0; i < 1000; ++i)
+            {
+                InputSystem.Update();
+                pos = gamepad.leftStick.value;
+                Assert.That(gamepad.leftStick.m_CachedValueIsStale, Is.False);
+                // Make sure there's a new different value every frames to mark the cached value as stale.
+                InputSystem.QueueStateEvent(gamepad, new GamepadState { leftStick = new Vector2(i / 1000f, i / 1000f) });
+            }
+        })
+            .MeasurementCount(100)
+            .WarmupCount(10)
+            .Run();
+    }
+
+    [Test, Performance]
+    [Category("Performance")]
+    [TestCase(OptimizationTestType.NoOptimization)]
+    [TestCase(OptimizationTestType.OptimizedControls)]
+    [TestCase(OptimizationTestType.ReadValueCaching)]
+    [TestCase(OptimizationTestType.OptimizedControlsAndReadValueCaching)]
+    // These tests evaluate the performance when there's no read value performed and only InputSystem.Update() is called.
+    // Emulates a scenario where the controls are not being changed to evaluate the impact of the optimizations.
+    public void Performance_OptimizedControls_UpdateOnly1kTimes(OptimizationTestType testType)
+    {
+        SetInternalFeatureFlagsFromTestType(testType);
+
+        // This adds FastMouse, which updates state every frame and can lead to a performance cost
+        // when using ReadValueCaching.
+        var mouse = InputSystem.AddDevice<Mouse>();
+        InputSystem.Update();
+
+        Measure.Method(() =>
+        {
+            CallUpdate();
+        })
+            .MeasurementCount(100)
+            .SampleGroup("Mouse Only")
+            .WarmupCount(10)
+            .Run();
+
+        InputSystem.RemoveDevice(mouse);
+        InputSystem.AddDevice<Gamepad>();
+        InputSystem.Update();
+
+        Measure.Method(() =>
+        {
+            CallUpdate();
+        })
+            .MeasurementCount(100)
+            .SampleGroup("Gamepad Only")
+            .WarmupCount(10)
+            .Run();
+
+        return;
+
+        void CallUpdate()
+        {
+            for (var i = 0; i < 1000; ++i) InputSystem.Update();
+        }
+    }
+
 #if ENABLE_VR
     [Test, Performance]
     [Category("Performance")]
-    [TestCase(OptimizedControlsTest.OptimizedControls)]
-    [TestCase(OptimizedControlsTest.NormalControls)]
-    public void Performance_OptimizedControls_ReadingPose4kTimes(OptimizedControlsTest testSetup)
+    [TestCase(OptimizationTestType.NoOptimization)]
+    [TestCase(OptimizationTestType.OptimizedControls)]
+    [TestCase(OptimizationTestType.ReadValueCaching)]
+    [TestCase(OptimizationTestType.OptimizedControlsAndReadValueCaching)]
+    // Isolated tests for reading from XR Pose device to evaluate the performance of the optimizations.
+    // Does not take into account the performance of the InputSystem.Update() call.
+    public void Performance_OptimizedControls_ReadingPose4kTimes(OptimizationTestType testType)
     {
-        var useOptimizedControls = testSetup == OptimizedControlsTest.OptimizedControls;
-        InputSystem.settings.SetInternalFeatureFlag(InputFeatureNames.kUseOptimizedControls, useOptimizedControls);
-        InputSystem.settings.SetInternalFeatureFlag(InputFeatureNames.kUseReadValueCaching, useOptimizedControls);
-        InputSystem.settings.SetInternalFeatureFlag(InputFeatureNames.kParanoidReadValueCachingChecks, false);
+        SetInternalFeatureFlagsFromTestType(testType);
 
         runtime.ReportNewInputDevice(XRTests.PoseDeviceState.CreateDeviceDescription().ToJson());
 
@@ -598,6 +786,8 @@ public void Performance_OptimizedControls_ReadingPose4kTimes(OptimizedControlsTe
         var device = InputSystem.devices[0];
 
         var poseControl = device["posecontrol"] as UnityEngine.InputSystem.XR.PoseControl;
+        var useOptimizedControls = testType == OptimizationTestType.OptimizedControls
+            || testType == OptimizationTestType.OptimizedControlsAndReadValueCaching;
         Assert.That(poseControl.optimizedControlDataType, Is.EqualTo(useOptimizedControls ? InputStateBlock.FormatPose : InputStateBlock.FormatInvalid));
 
         Measure.Method(() =>
