Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright (C) 2016-2017 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
"use strict";
class Driver {
constructor(statusCell, triggerCell, triggerLink, magicCell, summaryCell, key)
{
this._benchmarks = new Map();
this._statusCell = statusCell;
this._triggerCell = triggerCell;
this._triggerLink = triggerLink;
this._magicCell = magicCell;
this._summary = new Stats(summaryCell, "summary");
this._key = key;
this._values = [];
this._names = [];
if (isInBrowser)
window[key] = this;
}
addBenchmark(benchmark)
{
this._benchmarks.set(benchmark, new Results(benchmark));
}
readyTrigger()
{
if (isInBrowser) {
this._triggerCell.addEventListener('click', this._triggerLink);
this._triggerCell.classList.add('ready');
}
}
disableTrigger()
{
if (isInBrowser) {
this._triggerCell.removeEventListener('click', this._triggerLink);
this._triggerCell.classList.remove('ready');
}
}
start(numIterations)
{
this.disableTrigger();
this._updateIterations();
this._summary.reset();
for (let [benchmark, results] of this._benchmarks)
results.reset();
this._isRunning = true;
this._startIterations = this._numIterations = numIterations;
this._iterator = null;
this._iterate();
}
reportResult(...args)
{
this._benchmarks.get(this._benchmark).reportResult(...args);
this._recomputeSummary();
this._iterate();
}
reportError(...args)
{
if (isInBrowser)
console.log("Error encountered: ", ...args);
this._benchmarks.get(this._benchmark).reportError(...args);
if (isInBrowser) {
this._statusCell.innerHTML = "Test failure \u2014 error in console";
this._statusCell.classList.add("failed");
} else
print("Test failure");
}
_recomputeSummary()
{
class Geomean {
constructor()
{
this._count = 0;
this._sum = 0;
}
add(value)
{
this._count++;
this._sum += Math.log(value);
}
get result()
{
return Math.exp(this._sum * (1 / this._count));
}
}
let statses = [];
for (let [benchmark, results] of this._benchmarks) {
for (let subResult of Results.subResults) {
statses.push(results[subResult]);
let val = results[subResult].valueForIteration(results[subResult].numIterations - 1);
if (val > 0 && benchmark.name == this._benchmark.name) {
this._values.push(val);
this._names.push(benchmark.name + "_" + subResult);
}
}
}
let numIterations = Math.min(...statses.map(stats => stats.numIterations));
let data = new Array(numIterations);
for (let i = 0; i < data.length; ++i)
data[i] = new Geomean();
for (let stats of statses) {
for (let i = 0; i < data.length; ++i)
data[i].add(stats.valueForIteration(i));
}
let geomeans = data.map(geomean => geomean.result);
if (geomeans.length == this._startIterations) {
for (let iter = 0; iter < geomeans.length; iter++) {
this._values.push(geomeans[iter]);
this._names.push("geomean");
}
}
this._summary.reset(...geomeans);
}
_iterate()
{
this._benchmark = this._iterator ? this._iterator.next().value : null;
if (!this._benchmark) {
if (!this._numIterations) {
if (isInBrowser) {
this._statusCell.innerHTML = "Restart";
this.readyTrigger();
if (typeof tpRecordTime !== "undefined")
tpRecordTime(this._values.join(','), 0, this._names.join(','));
this.sendResultsToRaptor();
} else
print("Success! Benchmark is now finished.");
return;
}
this._numIterations--;
this._updateIterations();
this._iterator = this._benchmarks.keys();
this._benchmark = this._iterator.next().value;
}
this._benchmarks.get(this._benchmark).reportRunning();
let benchmark = this._benchmark;
if (isInBrowser) {
window.setTimeout(() => {
if (!this._isRunning)
return;
this._magicCell.contentDocument.body.textContent = "";
this._magicCell.contentDocument.body.innerHTML = "<iframe id=\"magicFrame\" frameborder=\"0\">";
let magicFrame = this._magicCell.contentDocument.getElementById("magicFrame");
magicFrame.contentDocument.open();
magicFrame.contentDocument.write(
`<!DOCTYPE html><head><title>benchmark payload</title></head><body><script>` +
`window.onerror = top.${this._key}.reportError.bind(top.${this._key});\n` +
`function reportResult()\n` +
`{\n` +
` var driver = top.${this._key};\n` +
` driver.reportResult.apply(driver, arguments);\n` +
`}\n` +
`</script>\n` +
`${this._benchmark.code}</body></html>`);
}, 100);
} else {
Promise.resolve(20).then(() => {
if (!this._isRunning)
return;
try {
print(`Running... ${this._benchmark.name} ( ${this._numIterations + 1} to go)`);
benchmark.run();
print("\n");
} catch(e) {
print(e);
print(e.stack);
}
});
}
}
_updateIterations()
{
if (isInBrowser) {
this._statusCell.innerHTML = "Running Tests... " + ( this._startIterations - this._numIterations ) + "/" + this._startIterations;
}
}
sendResultsToRaptor()
{
// this contains all test names = [ "Air_firstIteration", "Air_averageWorstCase", ...]
var allNames = this._names;
// this contains all test metrics = [ 111, 83.5, 21.78894472361809, ...]
var allValues = this._values;
// this object will store name:[value1, value2, ...] pairs for the arrays above
var measuredValuesByFullName = {};
for (var i = 0, len = allNames.length; i < len; i++) {
if (measuredValuesByFullName[allNames[i]] === undefined) {
measuredValuesByFullName[allNames[i]] = [];
}
}
allNames.map(function(name, index) {
// now we save all the values for each test
// ex.: measuredValuesByFullName['Air_firstIteration'].push(111);
// measuredValuesByFullName['Air_firstIteration'].push(107);
measuredValuesByFullName[name].push(allValues[index]);
});
// delete the geomean array - this will be calculated by raptor
delete measuredValuesByFullName.geomean;
if (location.search === '?raptor') {
var _data = ['raptor-benchmark', 'ares6', measuredValuesByFullName];
console.log('ares6 source is about to post results to the raptor webext');
window.postMessage(_data, '*');
window.sessionStorage.setItem('benchmark_results', JSON.stringify(_data));
}
}
}