Skip to content

Commit

Permalink
Move to having a single fetchAlignedRunsFromServer
Browse files Browse the repository at this point in the history
  • Loading branch information
@gsnedders
gsnedders committed Aug 9, 2022
1 parent 47afa63 commit 128f5b2
Show file tree
Hide file tree
Showing 5 changed files with 81 additions and 224 deletions.
74 changes: 1 addition & 73 deletions browser-specific-failures.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,12 @@
* time.
*/

const fetch = require('node-fetch');
const fs = require('fs');
const flags = require('flags');
const Git = require('nodegit');
const lib = require('./lib');
const moment = require('moment');

const {advanceDateToSkipBadDataIfNecessary} = require('./bad-ranges');

flags.defineString('from', '2018-07-01', 'Starting date (inclusive)');
flags.defineString('to', moment().format('YYYY-MM-DD'),
'Ending date (exclusive)');
Expand All @@ -26,75 +23,6 @@ flags.defineBoolean('experimental', false,
'Calculate metrics for experimental runs.');
flags.parse();

const RUNS_URI = 'https://wpt.fyi/api/runs?aligned=true&max-count=1';

// Fetches aligned runs from the wpt.fyi server, between the |from| and |to|
// dates. If |experimental| is true fetch experimental runs, else stable runs.
// Returns a map of date to list of runs for that date (one per product)
//
// TODO: Known problem: there are periods of time, mostly mid-late 2018, where
// we ran both Safari 11.1 and 12.1, and the results are massively different.
// We should fetch multiple runs for each browser and have upgrade logic.
async function fetchAlignedRunsFromServer(products, from, to, experimental) {
const label = experimental ? 'experimental' : 'stable';
let params = `&label=master&label=${label}`;
for (const product of products) {
params += `&product=${product}`;
}
const runsUri = `${RUNS_URI}${params}`;

console.log(`Fetching aligned runs from ${from.format('YYYY-MM-DD')} ` +
`to ${to.format('YYYY-MM-DD')}`);

let cachedCount = 0;
const before = moment();
const alignedRuns = new Map();
while (from < to) {
const formattedFrom = from.format('YYYY-MM-DD');
from.add(1, 'days');
const formattedTo = from.format('YYYY-MM-DD');

// We advance the date (if necessary) before doing anything more, so that
// code later in the loop body can just 'continue' without checking.
from = advanceDateToSkipBadDataIfNecessary(from, experimental);

// Attempt to read the runs from the cache.
// TODO: Consider https://github.com/tidoust/fetch-filecache-for-crawling
let runs;
const cacheFile =
`cache/${label}-${products.join('-')}-runs-${formattedFrom}.json`;
try {
runs = JSON.parse(await fs.promises.readFile(cacheFile));
if (runs.length) {
cachedCount++;
}
} catch (e) {
// No cache hit; load from the server instead.
const url = `${runsUri}&from=${formattedFrom}&to=${formattedTo}`;
const response = await fetch(url);
// Many days do not have an aligned set of runs, but we always write to
// the cache to speed up future executions of this code.
runs = await response.json();
await fs.promises.writeFile(cacheFile, JSON.stringify(runs));
}

if (!runs.length) {
continue;
}

if (runs.length !== products.length) {
throw new Error(
`Fetched ${runs.length} runs, expected ${products.length}`);
}

alignedRuns.set(formattedFrom, runs);
}
const after = moment();
console.log(`Fetched ${alignedRuns.size} sets of runs in ` +
`${after - before} ms (${cachedCount} cached)`);

return alignedRuns;
}

async function main() {
// Sort the products so that output files are consistent.
Expand All @@ -111,7 +39,7 @@ async function main() {
const from = moment(flags.get('from'));
const to = moment(flags.get('to'));
const experimental = flags.get('experimental');
const alignedRuns = await fetchAlignedRunsFromServer(
const alignedRuns = await lib.runs.fetchAlignedRunsFromServer(
products, from, to, experimental);

// Verify that we have data for the fetched runs in the results-analysis-cache
Expand Down
80 changes: 1 addition & 79 deletions compat-2021/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,13 @@
// TODO: There's a lot of reused code from browser-specific-failures.js here,
// that could be put into lib/

const fetch = require('node-fetch');
const flags = require('flags');
const fs = require('fs');
const Git = require('nodegit');
const lib = require('../lib');
const moment = require('moment');
const path = require('path');

const {advanceDateToSkipBadDataIfNecessary} = require('../bad-ranges');

flags.defineStringList('products', ['chrome', 'firefox', 'safari'],
'Products to include (comma-separated)');
flags.defineString('from', '2018-07-01', 'Starting date (inclusive)');
Expand All @@ -32,81 +29,6 @@ const CATEGORIES = [
'position-sticky',
];

const RUNS_URI = 'https://wpt.fyi/api/runs?aligned=true&max-count=1';

// Fetches aligned runs from the wpt.fyi server, between the |from| and |to|
// dates. If |experimental| is true fetch experimental runs, else stable runs.
// Returns a map of date to list of runs for that date (one per product)
//
// TODO: Known problem: there are periods of time, mostly mid-late 2018, where
// we ran both Safari 11.1 and 12.1, and the results are massively different.
// We should fetch multiple runs for each browser and have upgrade logic.
async function fetchAlignedRunsFromServer(products, from, to, experimental) {
const label = experimental ? 'experimental' : 'stable';
let params = `&label=master&label=${label}`;
for (const product of products) {
params += `&product=${product}`;
}
const runsUri = `${RUNS_URI}${params}`;

console.log(`Fetching aligned runs from ${from.format('YYYY-MM-DD')} ` +
`to ${to.format('YYYY-MM-DD')}`);

let cachedCount = 0;
const before = moment();
const alignedRuns = new Map();
while (from < to) {
const formattedFrom = from.format('YYYY-MM-DD');
from.add(1, 'days');
const formattedTo = from.format('YYYY-MM-DD');

// We advance the date (if necessary) before doing anything more, so that
// code later in the loop body can just 'continue' without checking.
from = advanceDateToSkipBadDataIfNecessary(from, experimental);

// Attempt to read the runs from the cache.
// TODO: Consider https://github.com/tidoust/fetch-filecache-for-crawling
let runs;
const cacheFile = path.join(ROOT_DIR,
`cache/${label}-${products.join('-')}-runs-${formattedFrom}.json`);
try {
runs = JSON.parse(await fs.promises.readFile(cacheFile));
if (runs.length) {
cachedCount++;
}
} catch (e) {
let url = `${runsUri}&from=${formattedFrom}&to=${formattedTo}`;
// HACK: Handle WebKitGTK runs being delayed vs other runs by extending
// the search radius if WebKitGTK is being requested.
if (products.includes('webkitgtk')) {
// eslint-disable-next-line max-len
url = `${runsUri}&from=${formattedFrom}T00:00:00Z&to=${formattedTo}T23:59:59Z`;
}
const response = await fetch(url);
// Many days do not have an aligned set of runs, but we always write to
// the cache to speed up future executions of this code.
runs = await response.json();
await fs.promises.writeFile(cacheFile, JSON.stringify(runs));
}

if (!runs.length) {
continue;
}

if (runs.length !== products.length) {
throw new Error(
`Fetched ${runs.length} runs, expected ${products.length}`);
}

alignedRuns.set(formattedFrom, runs);
}
const after = moment();
console.log(`Fetched ${alignedRuns.size} sets of runs in ` +
`${after - before} ms (${cachedCount} cached)`);

return alignedRuns;
}

async function loadAllTestsSet(category) {
const filename = path.join(ROOT_DIR, 'compat-2021', category + '-tests.txt');
const contents = await fs.promises.readFile(filename, 'utf-8');
Expand Down Expand Up @@ -304,7 +226,7 @@ async function main() {
const from = moment(flags.get('from'));
const to = moment(flags.get('to'));
const experimental = flags.get('experimental');
const alignedRuns = await fetchAlignedRunsFromServer(
const alignedRuns = await lib.runs.fetchAlignedRunsFromServer(
products, from, to, experimental);

// Verify that we have data for the fetched runs in the results-analysis-cache
Expand Down
68 changes: 1 addition & 67 deletions interop-2022/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ const lib = require('../lib');
const moment = require('moment');
const path = require('path');

const {advanceDateToSkipBadDataIfNecessary} = require('../bad-ranges');

flags.defineStringList('products', ['chrome', 'firefox', 'safari'],
'Products to include (comma-separated)');
flags.defineString('from', '2022-01-01', 'Starting date (inclusive)');
Expand Down Expand Up @@ -48,8 +46,6 @@ const CATEGORIES = [
'interop-2022-webcompat',
];

const RUNS_URI = 'https://wpt.fyi/api/runs?aligned=true&max-count=1';

// All non-OK harness statuses. Any non-OK harness status should be investigated
// before being added to this list, so that we don't score tests in the wrong
// way because of a test or infrastructure issue.
Expand Down Expand Up @@ -91,68 +87,6 @@ const KNOWN_TEST_STATUSES = new Set([
'/html/semantics/interactive-elements/the-dialog-element/backdrop-receives-element-events.html',
]);

// Fetches aligned runs from the wpt.fyi server, between the |from| and |to|
// dates. If |experimental| is true fetch experimental runs, else stable runs.
// Returns a map of date to list of runs for that date (one per product)
async function fetchAlignedRunsFromServer(products, from, to, experimental) {
const label = experimental ? 'experimental' : 'stable';
let params = `&label=master&label=${label}`;
for (const product of products) {
params += `&product=${product}`;
}
const runsUri = `${RUNS_URI}${params}`;

console.log(`Fetching aligned runs from ${from.format('YYYY-MM-DD')} ` +
`to ${to.format('YYYY-MM-DD')}`);

let cachedCount = 0;
const before = moment();
const alignedRuns = new Map();
while (from < to) {
const formattedFrom = from.format('YYYY-MM-DD');
from.add(1, 'days');
const formattedTo = from.format('YYYY-MM-DD');

// We advance the date (if necessary) before doing anything more, so that
// code later in the loop body can just 'continue' without checking.
from = advanceDateToSkipBadDataIfNecessary(from, experimental);

// Attempt to read the runs from the cache.
// TODO: Consider https://github.com/tidoust/fetch-filecache-for-crawling
let runs;
const cacheFile = path.join(ROOT_DIR,
`cache/${label}-${products.join('-')}-runs-${formattedFrom}.json`);
try {
runs = JSON.parse(await fs.promises.readFile(cacheFile));
if (runs.length) {
cachedCount++;
}
} catch (e) {
const url = `${runsUri}&from=${formattedFrom}&to=${formattedTo}`;
const response = await fetch(url);
// Many days do not have an aligned set of runs, but we always write to
// the cache to speed up future executions of this code.
runs = await response.json();
await fs.promises.writeFile(cacheFile, JSON.stringify(runs));
}

if (!runs.length) {
continue;
}

if (runs.length !== products.length) {
throw new Error(
`Fetched ${runs.length} runs, expected ${products.length}`);
}

alignedRuns.set(formattedFrom, runs);
}
const after = moment();
console.log(`Fetched ${alignedRuns.size} sets of runs in ` +
`${after - before} ms (${cachedCount} cached)`);

return alignedRuns;
}

// Score a set of runs (independently) on a set of tests. The runs are presumed
// to be aligned in some way (i.e. they were all run at the same WPT SHA).
Expand Down Expand Up @@ -276,7 +210,7 @@ async function main() {
const from = moment(flags.get('from'));
const to = moment(flags.get('to'));
const experimental = flags.get('experimental');
const alignedRuns = await fetchAlignedRunsFromServer(
const alignedRuns = await lib.runs.fetchAlignedRunsFromServer(
products, from, to, experimental);

// Verify that we have data for the fetched runs in the results-analysis-cache
Expand Down
Loading

0 comments on commit 128f5b2

Please sign in to comment.