Merge pull request #1489 from rylev/adjust-historical-analysis

Handle Noise Better

Author: rylev

site/src/api.rs

@@ -194,6 +194,7 @@ pub mod comparison {
         pub profile: String,
         pub scenario: String,
         pub is_relevant: bool,
+        pub significance_threshold: f64,
         pub significance_factor: f64,
         pub statistics: (f64, f64),
     }

site/src/comparison.rs

@@ -135,6 +135,7 @@ pub async fn handle_compare(
             profile: comparison.profile.to_string(),
             scenario: comparison.scenario.to_string(),
             is_relevant: comparison.is_relevant(),
+            significance_threshold: comparison.significance_threshold(),
             significance_factor: comparison.significance_factor(),
             statistics: comparison.results,
         })
@@ -1015,8 +1016,7 @@ pub struct HistoricalDataMap {
 }
 
 impl HistoricalDataMap {
-    const NUM_PREVIOUS_COMMITS: usize = 100;
-    const MIN_PREVIOUS_COMMITS: usize = 50;
+    const NUM_PREVIOUS_COMMITS: usize = 30;
 
     async fn calculate(
         ctxt: &SiteCtxt,
@@ -1033,7 +1033,7 @@ impl HistoricalDataMap {
         ));
 
         // Return early if we don't have enough data for historical analysis
-        if previous_commits.len() < Self::MIN_PREVIOUS_COMMITS {
+        if previous_commits.len() < Self::NUM_PREVIOUS_COMMITS {
             return Ok(Self {
                 data: historical_data,
             });
@@ -1057,7 +1057,7 @@ impl HistoricalDataMap {
         }
 
         // Only retain test cases for which we have enough data to calculate variance.
-        historical_data.retain(|_, v| v.data.len() >= Self::MIN_PREVIOUS_COMMITS);
+        historical_data.retain(|_, v| v.data.len() >= Self::NUM_PREVIOUS_COMMITS);
 
         Ok(Self {
             data: historical_data,

site/static/compare/script.js

@@ -134,6 +134,7 @@ const app = Vue.createApp({
                             category: (benchmarkMap[c.benchmark] || {}).category || "secondary",
                             isRelevant: c.is_relevant,
                             significanceFactor: c.significance_factor,
+                            significanceThreshold: (c.significance_threshold * 100.0), // ensure the threshold is in %
                             datumA,
                             datumB,
                             percent,
@@ -152,7 +153,7 @@ const app = Vue.createApp({
         bootstrapTotals() {
             const a = this.data.a.bootstrap_total / 1e9;
             const b = this.data.b.bootstrap_total / 1e9;
-            return {a, b};
+            return { a, b };
         },
         bootstraps() {
             return Object.entries(this.data.a.bootstrap).map(e => {
@@ -355,14 +356,20 @@ app.component('test-cases-table', {
             <th>Profile</th>
             <th>Scenario</th>
             <th>% Change</th>
+            <th>
+                Significance Threshold<span class="tooltip">?
+                    <span class="tooltiptext">
+                        The minimum % change that is considered significant. The higher the significance threshold, the noisier a test case is.
+                        You can see <a href="https://github.com/rust-lang/rustc-perf/blob/master/docs/comparison-analysis.md#what-makes-a-test-result-significant">
+                        here</a> how the significance threshold is calculated.
+                    </span>
+                </span>
+            </th>
             <th>
                 Significance Factor<span class="tooltip">?
                     <span class="tooltiptext">
                         How much a particular result is over the significance threshold. A factor of 2.50x
-                        means
-                        the result is 2.5 times over the significance threshold. You can see <a
-                            href="https://github.com/rust-lang/rustc-perf/blob/master/docs/comparison-analysis.md#what-makes-a-test-result-significant">
-                            here</a> how the significance threshold is calculated.
+                        means the result is 2.5 times over the significance threshold.
                     </span>
                 </span>
             </th>
@@ -393,6 +400,9 @@ app.component('test-cases-table', {
                         </span>
                     </a>
                 </td>
+                <td>
+                    {{ testCase.significanceThreshold ? testCase.significanceThreshold.toFixed(2) + "%" : "-" }}
+                </td>
                 <td>
                     {{ testCase.significanceFactor ? testCase.significanceFactor.toFixed(2) + "x" : "-" }}
                 </td>
@@ -447,7 +457,7 @@ const SummaryRange = {
   [<SummaryPercentValue :value="range[0]" :padWidth="6" />, <SummaryPercentValue :value="range[1]" :padWidth="6" />]
 </div>
 <div v-else style="text-align: center;">-</div>
-`, components: {SummaryPercentValue}
+`, components: { SummaryPercentValue }
 };
 const SummaryCount = {
     props: {
@@ -510,7 +520,7 @@ app.component('summary-table', {
     </tbody>
 </table>
 `,
-    components: {SummaryRange, SummaryPercentValue, SummaryCount}
+    components: { SummaryRange, SummaryPercentValue, SummaryCount }
 });
 
 app.component("aggregations", {