Add measurePrepare param (#440)

Collect the suite prepare time as separate metric if param.measurePrepare is set.

Author: camillobruni

index.html

@@ -100,7 +100,7 @@ <h2>Non-standard Parameters</h2>
                         </div>
                         <div class="aggregated-metric-result">
                             <h2>Aggregate Metric</h2>
-                            <div id="geomean-chart"></div>
+                            <div id="aggregate-chart"></div>
                             <h2>Test Metrics Overview</h2>
                             <div id="tests-chart"></div>
                         </div>

resources/benchmark-runner.mjs

@@ -468,10 +468,10 @@ export class BenchmarkRunner {
 
     _appendIterationMetrics() {
         const getMetric = (name, unit = "ms") => this._metrics[name] || (this._metrics[name] = new Metric(name, unit));
-        const iterationTotalMetric = (i) => {
+        const iterationMetric = (i, name) => {
             if (i >= params.iterationCount)
                 throw new Error(`Requested iteration=${i} does not exist.`);
-            return getMetric(`Iteration-${i}-Total`);
+            return getMetric(`Iteration-${i}-${name}`);
         };
 
         const collectSubMetrics = (prefix, items, parent) => {
@@ -496,20 +496,34 @@ export class BenchmarkRunner {
             // Prepare all iteration metrics so they are listed at the end of
             // of the _metrics object, before "Total" and "Score".
             for (let i = 0; i < this._iterationCount; i++)
-                iterationTotalMetric(i).description = `Test totals for iteration ${i}`;
+                iterationMetric(i, "Total").description = `Test totals for iteration ${i}`;
             getMetric("Geomean", "ms").description = "Geomean of test totals";
             getMetric("Score", "score").description = "Scaled inverse of the Geomean";
+            if (params.measurePrepare)
+                getMetric("Prepare", "ms").description = "Geomean of workload prepare times";
         }
 
         const geomean = getMetric("Geomean");
-        const iterationTotal = iterationTotalMetric(geomean.length);
+        const iteration = geomean.length;
+        const iterationTotal = iterationMetric(iteration, "Total");
         for (const results of Object.values(iterationResults))
             iterationTotal.add(results.total);
         iterationTotal.computeAggregatedMetrics();
         geomean.add(iterationTotal.geomean);
         getMetric("Score").add(geomeanToScore(iterationTotal.geomean));
 
+        if (params.measurePrepare) {
+            const iterationPrepare = iterationMetric(iteration, "Prepare");
+            for (const results of Object.values(iterationResults))
+                iterationPrepare.add(results.prepare);
+            iterationPrepare.computeAggregatedMetrics();
+            const prepare = getMetric("Prepare");
+            prepare.add(iterationPrepare.geomean);
+        }
+
         for (const metric of Object.values(this._metrics))
             metric.computeAggregatedMetrics();
     }
+
+    _initializeMetrics() {}
 }

resources/developer-mode.mjs

@@ -18,6 +18,7 @@ export function createDeveloperModeContainer() {
     const settings = document.createElement("div");
     settings.className = "settings";
     settings.append(createUIForIterationCount());
+    settings.append(createUIForMeasurePrepare());
     settings.append(createUIForWarmupSuite());
     settings.append(createUIForWarmupBeforeSync());
     settings.append(createUIForSyncStepDelay());
@@ -46,6 +47,12 @@ function createUIForWarmupSuite() {
     });
 }
 
+function createUIForMeasurePrepare() {
+    return createCheckboxUI("Measure Prepare", params.measurePrepare, (isChecked) => {
+        params.measurePrepare = isChecked;
+    });
+}
+
 function createUIForAsyncSteps() {
     return createCheckboxUI("Use Async Steps", params.useAsyncSteps, (isChecked) => {
         params.useAsyncSteps = isChecked;

resources/main.mjs

@@ -250,8 +250,11 @@ class MainBenchmarkClient {
         const trackHeight = 24;
         document.documentElement.style.setProperty("--metrics-line-height", `${trackHeight}px`);
         const plotWidth = (params.viewport.width - 120) / 2;
-        document.getElementById("geomean-chart").innerHTML = renderMetricView({
-            metrics: [metrics.Geomean],
+        const aggregateMetrics = [metrics.Geomean];
+        if (params.measurePrepare)
+            aggregateMetrics.push(metrics.Prepare);
+        document.getElementById("aggregate-chart").innerHTML = renderMetricView({
+            metrics: aggregateMetrics,
             width: plotWidth,
             trackHeight,
             renderChildren: false,

resources/shared/benchmark.mjs

@@ -44,6 +44,7 @@ export class BenchmarkSuite {
     async runAndRecord(params, onProgress) {
         const measuredValues = {
             tests: {},
+            prepare: 0,
             total: 0,
         };
         const suiteStartLabel = `suite-${this.name}-start`;

resources/shared/params.mjs

@@ -27,6 +27,8 @@ export class Params {
     // "generate": generate a random seed
     // <integer>: use the provided integer as a seed
     shuffleSeed = "off";
+    // Measure more workload prepare time.
+    measurePrepare = false;
 
     constructor(searchParams = undefined) {
         if (searchParams)
@@ -57,6 +59,7 @@ export class Params {
         this.warmupBeforeSync = this._parseIntParam(searchParams, "warmupBeforeSync", 0);
         this.measurementMethod = this._parseMeasurementMethod(searchParams);
         this.shuffleSeed = this._parseShuffleSeed(searchParams);
+        this.measurePrepare = this._parseBooleanParam(searchParams, "measurePrepare");
 
         const unused = Array.from(searchParams.keys());
         if (unused.length > 0)

resources/suite-runner.mjs

@@ -8,12 +8,13 @@ export class SuiteRunner {
     #suite;
     #client;
     #suiteResults;
+    #prepareTime = 0;
 
     constructor(frame, page, params, suite, client, measuredValues) {
         // FIXME: Create SuiteRunner-local measuredValues.
         this.#suiteResults = measuredValues.tests[suite.name];
         if (!this.#suiteResults) {
-            this.#suiteResults = { tests: {}, total: 0 };
+            this.#suiteResults = { tests: {}, prepare: 0, total: 0 };
             measuredValues.tests[suite.name] = this.#suiteResults;
         }
         this.#frame = frame;
@@ -62,7 +63,8 @@ export class SuiteRunner {
         await this.#suite.prepare(this.#page);
         performance.mark(suitePrepareEndLabel);
 
-        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+        const entry = performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+        this.#prepareTime = entry.duration;
     }
 
     async _runSuite() {
@@ -83,17 +85,19 @@ export class SuiteRunner {
         performance.mark(suiteEndLabel);
 
         performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
-        this._validateSuiteTotal();
+        this._validateSuiteResults();
         await this._updateClient();
     }
 
-    _validateSuiteTotal() {
+    _validateSuiteResults() {
         // When the test is fast and the precision is low (for example with Firefox'
         // privacy.resistFingerprinting preference), it's possible that the measured
         // total duration for an entire is 0.
-        const suiteTotal = this.#suiteResults.total;
+        const { suiteTotal, suitePrepare } = this.#suiteResults.total;
         if (suiteTotal === 0)
             throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`);
+        if (this.#params.measurePrepare && suitePrepare === 0)
+            throw new Error(`Got invalid 0-time prepare time for suite ${this.#suite.name}: ${suitePrepare}`);
     }
 
     async _loadFrame() {
@@ -110,9 +114,13 @@ export class SuiteRunner {
         if (this.#suite === WarmupSuite)
             return;
 
-        const total = syncTime + asyncTime;
-        this.#suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
-        this.#suiteResults.total += total;
+        let total = syncTime + asyncTime;
+        this.#suiteResults.tests[test.name] = {
+            tests: { Sync: syncTime, Async: asyncTime },
+            total: total,
+        };
+        this.#suiteResults.prepare = this.#prepareTime;
+        this.#suiteResults.total = total;
     };
 
     async _updateClient(suite = this.#suite) {
@@ -123,6 +131,7 @@ export class SuiteRunner {
 
 export class RemoteSuiteRunner extends SuiteRunner {
     #appId;
+    #prepareTime;
 
     get appId() {
         return this.#appId;
@@ -163,7 +172,8 @@ export class RemoteSuiteRunner extends SuiteRunner {
 
         performance.mark(suitePrepareEndLabel);
 
-        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+        const entry = performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+        this.#prepareTime = entry.duration;
     }
 
     async _runSuite() {
@@ -177,9 +187,10 @@ export class RemoteSuiteRunner extends SuiteRunner {
             ...response.result.tests,
         };
 
-        this.suiteResults.total += response.result.total;
+        this.suiteResults.prepare = this.#prepareTime;
+        this.suiteResults.total = response.result.total;
 
-        this._validateSuiteTotal();
+        this._validateSuiteResults();
         await this._updateClient();
     }
 

tests/unittests/benchmark-runner.mjs

@@ -138,15 +138,15 @@ describe("BenchmarkRunner", () => {
         });
 
         describe("runSuite", () => {
-            let _prepareSuiteSpy, _loadFrameStub, _runTestStub, _validateSuiteTotalStub, _suitePrepareSpy, performanceMarkSpy;
+            let _prepareSuiteSpy, _loadFrameStub, _runTestStub, _validateSuiteRessultsStub, _suitePrepareSpy, performanceMarkSpy;
 
             const suite = SUITES_FIXTURE[0];
 
             before(async () => {
                 _prepareSuiteSpy = spy(SuiteRunner.prototype, "_prepareSuite");
                 _loadFrameStub = stub(SuiteRunner.prototype, "_loadFrame").callsFake(async () => null);
                 _runTestStub = stub(TestRunner.prototype, "runTest").callsFake(async () => null);
-                _validateSuiteTotalStub = stub(SuiteRunner.prototype, "_validateSuiteTotal").callsFake(async () => null);
+                _validateSuiteRessultsStub = stub(SuiteRunner.prototype, "_validateSuiteResults").callsFake(async () => null);
                 performanceMarkSpy = spy(window.performance, "mark");
                 _suitePrepareSpy = spy(suite, "prepare");
 
@@ -161,7 +161,7 @@ describe("BenchmarkRunner", () => {
 
             it("should run and record results for every test in suite", async () => {
                 assert.calledThrice(_runTestStub);
-                assert.calledOnce(_validateSuiteTotalStub);
+                assert.calledOnce(_validateSuiteRessultsStub);
                 assert.calledWith(performanceMarkSpy, "suite-Suite 1-prepare-start");
                 assert.calledWith(performanceMarkSpy, "suite-Suite 1-prepare-end");
                 assert.calledWith(performanceMarkSpy, "suite-Suite 1-start");