Merge pull request #1277 from rylev/relevance-filter

Change the 'significance' filter on the comparison page to a 'relevance' filter

Author: rylev

site/src/api.rs

@@ -189,9 +189,8 @@ pub mod comparison {
         pub benchmark: String,
         pub profile: String,
         pub scenario: String,
-        pub is_significant: bool,
+        pub is_relevant: bool,
         pub significance_factor: Option<f64>,
-        pub magnitude: String,
         pub statistics: (f64, f64),
     }
 }

site/src/comparison.rs

@@ -117,9 +117,8 @@ pub async fn handle_compare(
             benchmark: comparison.benchmark.to_string(),
             profile: comparison.profile.to_string(),
             scenario: comparison.scenario.to_string(),
-            is_significant: comparison.is_significant(),
+            is_relevant: comparison.is_relevant(),
             significance_factor: comparison.significance_factor(),
-            magnitude: comparison.magnitude().display().to_owned(),
             statistics: comparison.results,
         })
         .collect();
@@ -1142,16 +1141,6 @@ impl Magnitude {
     fn is_medium_or_above(&self) -> bool {
         *self >= Self::Medium
     }
-
-    pub fn display(&self) -> &'static str {
-        match self {
-            Self::VerySmall => "very small",
-            Self::Small => "small",
-            Self::Medium => "moderate",
-            Self::Large => "large",
-            Self::VeryLarge => "very large",
-        }
-    }
 }
 
 async fn generate_report(

site/static/compare.html

@@ -525,29 +525,17 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                     </ul>
                 </div>
                 <div class="section">
-                    <div class="section-heading"><span>Show only significant changes</span>
+                    <div class="section-heading"><span>Show non-relevant results</span>
                         <span class="tooltip">?
                             <span class="tooltiptext">
-                                Whether to filter out all benchmarks that do not show significant changes. A significant
-                                change is any change greater than or equal to 0.2% for non-noisy benchmarks and above
-                                1.0% for noisy ones.
+                                Whether to show test case results that are not relevant (i.e., not significant or 
+                                have a large enough magnitude). You can see 
+                                <a href="https://github.com/rust-lang/rustc-perf/blob/master/docs/comparison-analysis.md#how-is-relevance-of-a-test-run-summary-determined">
+                                here</a> how relevance is calculated.
                             </span>
                         </span>
                     </div>
-                    <input type="checkbox" v-model="filter.showOnlySignificant" style="margin-left: 20px;" />
-                </div>
-                <div class="section">
-                    <div class="section-heading"><span>Filter out very small changes</span>
-                        <span class="tooltip">?
-                            <span class="tooltiptext">
-                                Whether to filter out test cases that have a very small magnitude. Magnitude is
-                                calculated both on the absolute magnitude (i.e., how large of a percentage change)
-                                as well as the magnitude of the significance (i.e., by how many time the change was
-                                over the significance threshold).
-                            </span>
-                        </span>
-                    </div>
-                    <input type="checkbox" v-model="filter.filterVerySmall" style="margin-left: 20px;" />
+                    <input type="checkbox" v-model="filter.showNonRelevant" style="margin-left: 20px;" />
                 </div>
                 <div class="section">
                     <div class="section-heading"><span>Display raw data</span>
@@ -700,8 +688,7 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                 return {
                     filter: {
                         name: null,
-                        showOnlySignificant: true,
-                        filterVerySmall: true,
+                        showNonRelevant: false,
                         profile: {
                             check: true,
                             debug: true,
@@ -782,17 +769,14 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                         let nameFilter = filter.name && filter.name.trim();
                         nameFilter = !nameFilter || name.includes(nameFilter);
 
-                        const significanceFilter = filter.showOnlySignificant ? testCase.isSignificant : true;
-
-                        const magnitudeFilter = filter.filterVerySmall ? testCase.magnitude != "very small" : true;
+                        const relevanceFilter = filter.showNonRelevant ? true : testCase.isRelevant;
 
                         return (
                             profileFilter(testCase.profile) &&
                             scenarioFilter(testCase.scenario) &&
                             categoryFilter(testCase.category) &&
-                            significanceFilter &&
-                            nameFilter &&
-                            magnitudeFilter
+                            relevanceFilter &&
+                            nameFilter 
                         );
                     }
 
@@ -807,8 +791,7 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                                     profile: c.profile,
                                     scenario: c.scenario,
                                     category: (benchmarkMap[c.benchmark] || {}).category || "secondary",
-                                    magnitude: c.magnitude,
-                                    isSignificant: c.is_significant,
+                                    isRelevant: c.is_relevant,
                                     significanceFactor: c.significance_factor,
                                     datumA,
                                     datumB,
@@ -909,10 +892,10 @@ <h2>Comparing <span id="stat-header">{{stat}}</span> between <span id="before">{
                     };
 
                     const addDatum = (result, datum, percent) => {
-                        if (percent > 0 && datum.is_significant) {
+                        if (percent > 0 && datum.is_relevant) {
                             result.regressions += 1;
                             result.regressions_avg += percent;
-                        } else if (percent < 0 && datum.is_significant) {
+                        } else if (percent < 0 && datum.is_relevant) {
                             result.improvements += 1;
                             result.improvements_avg += percent;
                         } else {