Use more helper runner helper methods (#421)

Refactor the runner and add more helper methods to be more flexible for upcoming experimental PRs.

Author: camillobruni

resources/benchmark-runner.mjs

@@ -362,7 +362,7 @@ export class BenchmarkRunner {
         const iterationEndLabel = "iteration-end";
         for (let i = 0; i < this._iterationCount; i++) {
             performance.mark(iterationStartLabel);
-            await this._runAllSuites();
+            await this.runAllSuites();
             performance.mark(iterationEndLabel);
             performance.measure(`iteration-${i}`, iterationStartLabel, iterationEndLabel);
         }
@@ -375,7 +375,7 @@ export class BenchmarkRunner {
         }
     }
 
-    async _appendFrame(src) {
+    async _appendFrame() {
         const frame = document.createElement("iframe");
         const style = frame.style;
         style.width = `${params.viewport.width}px`;
@@ -396,7 +396,7 @@ export class BenchmarkRunner {
         return frame;
     }
 
-    async _runAllSuites() {
+    async _prepareAllSuites() {
         this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN };
 
         const prepareStartLabel = "runner-prepare-start";
@@ -408,23 +408,32 @@ export class BenchmarkRunner {
         this._page = new Page(this._frame);
 
         let suites = [...this._suites];
-        if (this._suiteOrderRandomNumberGenerator) {
-            // We just do a simple Fisher-Yates shuffle based on the repeated hash of the
-            // seed. This is not a high quality RNG, but it's plenty good enough.
-            for (let i = 0; i < suites.length - 1; i++) {
-                let j = i + (this._suiteOrderRandomNumberGenerator() % (suites.length - i));
-                let tmp = suites[i];
-                suites[i] = suites[j];
-                suites[j] = tmp;
-            }
-        }
+        if (this._suiteOrderRandomNumberGenerator)
+            this._shuffleSuites(suites);
+
         performance.mark(prepareEndLabel);
         performance.measure("runner-prepare", prepareStartLabel, prepareEndLabel);
 
+        return suites;
+    }
+
+    _shuffleSuites(suites) {
+        // We just do a simple Fisher-Yates shuffle based on the repeated hash of the
+        // seed. This is not a high quality RNG, but it's plenty good enough.
+        for (let i = 0; i < suites.length - 1; i++) {
+            const j = i + (this._suiteOrderRandomNumberGenerator() % (suites.length - i));
+            const tmp = suites[i];
+            suites[i] = suites[j];
+            suites[j] = tmp;
+        }
+    }
+
+    async runAllSuites() {
+        const suites = await this._prepareAllSuites();
         try {
             for (const suite of suites) {
                 if (!suite.disabled)
-                    await this._runSuite(suite);
+                    await this.runSuite(suite);
             }
 
         } finally {
@@ -444,23 +453,34 @@ export class BenchmarkRunner {
         performance.measure("runner-finalize", finalizeStartLabel, finalizeEndLabel);
     }
 
-    async _runSuite(suite) {
+    async runSuite(suite) {
+        await this._prepareSuite(suite);
+        await this._runSuite(suite);
+    }
+
+    async _prepareSuite(suite) {
         const suiteName = suite.name;
         const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
         const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
-        const suiteStartLabel = `suite-${suiteName}-start`;
-        const suiteEndLabel = `suite-${suiteName}-end`;
 
         performance.mark(suitePrepareStartLabel);
-        await this._prepareSuite(suite);
+        await this._loadFrame(suite);
+        await suite.prepare(this._page);
         performance.mark(suitePrepareEndLabel);
 
+        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+    }
+
+    async _runSuite(suite) {
+        const suiteName = suite.name;
+        const suiteStartLabel = `suite-${suiteName}-start`;
+        const suiteEndLabel = `suite-${suiteName}-end`;
+
         performance.mark(suiteStartLabel);
         for (const test of suite.tests)
             await this._runTestAndRecordResults(suite, test);
         performance.mark(suiteEndLabel);
 
-        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
         performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
         this._validateSuiteTotal(suiteName);
     }
@@ -474,14 +494,12 @@ export class BenchmarkRunner {
             throw new Error(`Got invalid 0-time total for suite ${suiteName}: ${suiteTotal}`);
     }
 
-    async _prepareSuite(suite) {
-        return new Promise((resolve) => {
+    async _loadFrame(suite) {
+        return new Promise((resolve, reject) => {
             const frame = this._page._frame;
-            frame.onload = async () => {
-                await suite.prepare(this._page);
-                resolve();
-            };
-            frame.src = `${suite.url}`;
+            frame.onload = () => resolve();
+            frame.onerror = () => reject();
+            frame.src = suite.url;
         });
     }
 

tests/benchmark-runner-tests.mjs

@@ -11,10 +11,12 @@ function TEST_FIXTURE(name) {
 const SUITES_FIXTURE = [
     {
         name: "Suite 1",
+        async prepare(page) {},
         tests: [TEST_FIXTURE("Test 1"), TEST_FIXTURE("Test 2"), TEST_FIXTURE("Test 3")],
     },
     {
         name: "Suite 2",
+        async prepare(page) {},
         tests: [TEST_FIXTURE("Test 1")],
     },
 ];
@@ -106,22 +108,37 @@ describe("BenchmarkRunner", () => {
     });
 
     describe("Suite", () => {
-        describe("_runAllSuites", () => {
-            let _runSuiteStub, _finalizeStub, _removeFrameStub;
+        describe("runAllSuites", () => {
+            let _runSuiteStub, _finalizeStub, _loadFrameStub, _appendFrameStub, _removeFrameStub;
 
             before(async () => {
-                _runSuiteStub = stub(runner, "_runSuite").callsFake(() => null);
-                _finalizeStub = stub(runner, "_finalize").callsFake(() => null);
+                _runSuiteStub = stub(runner, "_runSuite").callsFake(async () => null);
+                _finalizeStub = stub(runner, "_finalize").callsFake(async () => null);
+                _loadFrameStub = stub(runner, "_loadFrame").callsFake(async () => null);
+                _appendFrameStub = stub(runner, "_appendFrame").callsFake(async () => null);
                 _removeFrameStub = stub(runner, "_removeFrame").callsFake(() => null);
+                for (const suite of runner._suites)
+                    spy(suite, "prepare");
+                expect(runner._suites).not.to.have.length(0);
+                await runner.runAllSuites();
+            });
 
-                await runner._runAllSuites();
+            it("should call prepare on all suites", () => {
+                let suitesPrepareCount = 0;
+                for (const suite of runner._suites) {
+                    suitesPrepareCount += 1;
+                    assert.calledOnce(suite.prepare);
+                }
+                expect(suitesPrepareCount).equal(SUITES_FIXTURE.length);
             });
 
             it("should run all test suites", async () => {
                 assert.calledTwice(_runSuiteStub);
             });
 
             it("should remove the previous frame and then the current frame", () => {
+                assert.calledTwice(_loadFrameStub);
+                assert.calledOnce(_appendFrameStub);
                 assert.calledTwice(_removeFrameStub);
             });
 
@@ -130,22 +147,25 @@ describe("BenchmarkRunner", () => {
             });
         });
 
-        describe("_runSuite", () => {
-            let _prepareSuiteStub, _runTestAndRecordResultsStub, performanceMarkSpy;
+        describe("runSuite", () => {
+            let _prepareSuiteSpy, _loadFrameStub, _runTestAndRecordResultsStub, _suitePrepareSpy, performanceMarkSpy;
 
             const suite = SUITES_FIXTURE[0];
 
             before(async () => {
-                _prepareSuiteStub = stub(runner, "_prepareSuite").callsFake(() => null);
-
-                _runTestAndRecordResultsStub = stub(runner, "_runTestAndRecordResults").callsFake(() => null);
-
+                _prepareSuiteSpy = spy(runner, "_prepareSuite");
+                _loadFrameStub = stub(runner, "_loadFrame").callsFake(async () => null);
+                _runTestAndRecordResultsStub = stub(runner, "_runTestAndRecordResults").callsFake(async () => null);
                 performanceMarkSpy = spy(window.performance, "mark");
-                runner._runSuite(suite);
+                _suitePrepareSpy = spy(suite, "prepare");
+
+                runner.runSuite(suite);
             });
 
             it("should prepare the suite first", async () => {
-                assert.calledOnce(_prepareSuiteStub);
+                assert.calledOnce(_prepareSuiteSpy);
+                assert.calledOnce(_suitePrepareSpy);
+                assert.calledOnce(_loadFrameStub);
             });
 
             it("should run and record results for every test in suite", async () => {