diff --git a/browser-specific-failures.js b/browser-specific-failures.js index 5619400df..53117bb41 100644 --- a/browser-specific-failures.js +++ b/browser-specific-failures.js @@ -5,15 +5,12 @@ * time. */ -const fetch = require('node-fetch'); const fs = require('fs'); const flags = require('flags'); const Git = require('nodegit'); const lib = require('./lib'); const moment = require('moment'); -const {advanceDateToSkipBadDataIfNecessary} = require('./bad-ranges'); - flags.defineString('from', '2018-07-01', 'Starting date (inclusive)'); flags.defineString('to', moment().format('YYYY-MM-DD'), 'Ending date (exclusive)'); @@ -26,75 +23,6 @@ flags.defineBoolean('experimental', false, 'Calculate metrics for experimental runs.'); flags.parse(); -const RUNS_URI = 'https://wpt.fyi/api/runs?aligned=true&max-count=1'; - -// Fetches aligned runs from the wpt.fyi server, between the |from| and |to| -// dates. If |experimental| is true fetch experimental runs, else stable runs. -// Returns a map of date to list of runs for that date (one per product) -// -// TODO: Known problem: there are periods of time, mostly mid-late 2018, where -// we ran both Safari 11.1 and 12.1, and the results are massively different. -// We should fetch multiple runs for each browser and have upgrade logic. -async function fetchAlignedRunsFromServer(products, from, to, experimental) { - const label = experimental ? 'experimental' : 'stable'; - let params = `&label=master&label=${label}`; - for (const product of products) { - params += `&product=${product}`; - } - const runsUri = `${RUNS_URI}${params}`; - - console.log(`Fetching aligned runs from ${from.format('YYYY-MM-DD')} ` + - `to ${to.format('YYYY-MM-DD')}`); - - let cachedCount = 0; - const before = moment(); - const alignedRuns = new Map(); - while (from < to) { - const formattedFrom = from.format('YYYY-MM-DD'); - from.add(1, 'days'); - const formattedTo = from.format('YYYY-MM-DD'); - - // We advance the date (if necessary) before doing anything more, so that - // code later in the loop body can just 'continue' without checking. - from = advanceDateToSkipBadDataIfNecessary(from, experimental); - - // Attempt to read the runs from the cache. - // TODO: Consider https://github.com/tidoust/fetch-filecache-for-crawling - let runs; - const cacheFile = - `cache/${label}-${products.join('-')}-runs-${formattedFrom}.json`; - try { - runs = JSON.parse(await fs.promises.readFile(cacheFile)); - if (runs.length) { - cachedCount++; - } - } catch (e) { - // No cache hit; load from the server instead. - const url = `${runsUri}&from=${formattedFrom}&to=${formattedTo}`; - const response = await fetch(url); - // Many days do not have an aligned set of runs, but we always write to - // the cache to speed up future executions of this code. - runs = await response.json(); - await fs.promises.writeFile(cacheFile, JSON.stringify(runs)); - } - - if (!runs.length) { - continue; - } - - if (runs.length !== products.length) { - throw new Error( - `Fetched ${runs.length} runs, expected ${products.length}`); - } - - alignedRuns.set(formattedFrom, runs); - } - const after = moment(); - console.log(`Fetched ${alignedRuns.size} sets of runs in ` + - `${after - before} ms (${cachedCount} cached)`); - - return alignedRuns; -} async function main() { // Sort the products so that output files are consistent. @@ -111,7 +39,7 @@ async function main() { const from = moment(flags.get('from')); const to = moment(flags.get('to')); const experimental = flags.get('experimental'); - const alignedRuns = await fetchAlignedRunsFromServer( + const alignedRuns = await lib.runs.fetchAlignedRunsFromServer( products, from, to, experimental); // Verify that we have data for the fetched runs in the results-analysis-cache diff --git a/compat-2021/main.js b/compat-2021/main.js index 3964ec304..0a84cb0d7 100644 --- a/compat-2021/main.js +++ b/compat-2021/main.js @@ -3,7 +3,6 @@ // TODO: There's a lot of reused code from browser-specific-failures.js here, // that could be put into lib/ -const fetch = require('node-fetch'); const flags = require('flags'); const fs = require('fs'); const Git = require('nodegit'); @@ -11,8 +10,6 @@ const lib = require('../lib'); const moment = require('moment'); const path = require('path'); -const {advanceDateToSkipBadDataIfNecessary} = require('../bad-ranges'); - flags.defineStringList('products', ['chrome', 'firefox', 'safari'], 'Products to include (comma-separated)'); flags.defineString('from', '2018-07-01', 'Starting date (inclusive)'); @@ -32,81 +29,6 @@ const CATEGORIES = [ 'position-sticky', ]; -const RUNS_URI = 'https://wpt.fyi/api/runs?aligned=true&max-count=1'; - -// Fetches aligned runs from the wpt.fyi server, between the |from| and |to| -// dates. If |experimental| is true fetch experimental runs, else stable runs. -// Returns a map of date to list of runs for that date (one per product) -// -// TODO: Known problem: there are periods of time, mostly mid-late 2018, where -// we ran both Safari 11.1 and 12.1, and the results are massively different. -// We should fetch multiple runs for each browser and have upgrade logic. -async function fetchAlignedRunsFromServer(products, from, to, experimental) { - const label = experimental ? 'experimental' : 'stable'; - let params = `&label=master&label=${label}`; - for (const product of products) { - params += `&product=${product}`; - } - const runsUri = `${RUNS_URI}${params}`; - - console.log(`Fetching aligned runs from ${from.format('YYYY-MM-DD')} ` + - `to ${to.format('YYYY-MM-DD')}`); - - let cachedCount = 0; - const before = moment(); - const alignedRuns = new Map(); - while (from < to) { - const formattedFrom = from.format('YYYY-MM-DD'); - from.add(1, 'days'); - const formattedTo = from.format('YYYY-MM-DD'); - - // We advance the date (if necessary) before doing anything more, so that - // code later in the loop body can just 'continue' without checking. - from = advanceDateToSkipBadDataIfNecessary(from, experimental); - - // Attempt to read the runs from the cache. - // TODO: Consider https://github.com/tidoust/fetch-filecache-for-crawling - let runs; - const cacheFile = path.join(ROOT_DIR, - `cache/${label}-${products.join('-')}-runs-${formattedFrom}.json`); - try { - runs = JSON.parse(await fs.promises.readFile(cacheFile)); - if (runs.length) { - cachedCount++; - } - } catch (e) { - let url = `${runsUri}&from=${formattedFrom}&to=${formattedTo}`; - // HACK: Handle WebKitGTK runs being delayed vs other runs by extending - // the search radius if WebKitGTK is being requested. - if (products.includes('webkitgtk')) { - // eslint-disable-next-line max-len - url = `${runsUri}&from=${formattedFrom}T00:00:00Z&to=${formattedTo}T23:59:59Z`; - } - const response = await fetch(url); - // Many days do not have an aligned set of runs, but we always write to - // the cache to speed up future executions of this code. - runs = await response.json(); - await fs.promises.writeFile(cacheFile, JSON.stringify(runs)); - } - - if (!runs.length) { - continue; - } - - if (runs.length !== products.length) { - throw new Error( - `Fetched ${runs.length} runs, expected ${products.length}`); - } - - alignedRuns.set(formattedFrom, runs); - } - const after = moment(); - console.log(`Fetched ${alignedRuns.size} sets of runs in ` + - `${after - before} ms (${cachedCount} cached)`); - - return alignedRuns; -} - async function loadAllTestsSet(category) { const filename = path.join(ROOT_DIR, 'compat-2021', category + '-tests.txt'); const contents = await fs.promises.readFile(filename, 'utf-8'); @@ -304,7 +226,7 @@ async function main() { const from = moment(flags.get('from')); const to = moment(flags.get('to')); const experimental = flags.get('experimental'); - const alignedRuns = await fetchAlignedRunsFromServer( + const alignedRuns = await lib.runs.fetchAlignedRunsFromServer( products, from, to, experimental); // Verify that we have data for the fetched runs in the results-analysis-cache diff --git a/interop-2022/main.js b/interop-2022/main.js index 5b34d0601..6068f72db 100644 --- a/interop-2022/main.js +++ b/interop-2022/main.js @@ -17,8 +17,6 @@ const lib = require('../lib'); const moment = require('moment'); const path = require('path'); -const {advanceDateToSkipBadDataIfNecessary} = require('../bad-ranges'); - flags.defineStringList('products', ['chrome', 'firefox', 'safari'], 'Products to include (comma-separated)'); flags.defineString('from', '2022-01-01', 'Starting date (inclusive)'); @@ -48,8 +46,6 @@ const CATEGORIES = [ 'interop-2022-webcompat', ]; -const RUNS_URI = 'https://wpt.fyi/api/runs?aligned=true&max-count=1'; - // All non-OK harness statuses. Any non-OK harness status should be investigated // before being added to this list, so that we don't score tests in the wrong // way because of a test or infrastructure issue. @@ -91,68 +87,6 @@ const KNOWN_TEST_STATUSES = new Set([ '/html/semantics/interactive-elements/the-dialog-element/backdrop-receives-element-events.html', ]); -// Fetches aligned runs from the wpt.fyi server, between the |from| and |to| -// dates. If |experimental| is true fetch experimental runs, else stable runs. -// Returns a map of date to list of runs for that date (one per product) -async function fetchAlignedRunsFromServer(products, from, to, experimental) { - const label = experimental ? 'experimental' : 'stable'; - let params = `&label=master&label=${label}`; - for (const product of products) { - params += `&product=${product}`; - } - const runsUri = `${RUNS_URI}${params}`; - - console.log(`Fetching aligned runs from ${from.format('YYYY-MM-DD')} ` + - `to ${to.format('YYYY-MM-DD')}`); - - let cachedCount = 0; - const before = moment(); - const alignedRuns = new Map(); - while (from < to) { - const formattedFrom = from.format('YYYY-MM-DD'); - from.add(1, 'days'); - const formattedTo = from.format('YYYY-MM-DD'); - - // We advance the date (if necessary) before doing anything more, so that - // code later in the loop body can just 'continue' without checking. - from = advanceDateToSkipBadDataIfNecessary(from, experimental); - - // Attempt to read the runs from the cache. - // TODO: Consider https://github.com/tidoust/fetch-filecache-for-crawling - let runs; - const cacheFile = path.join(ROOT_DIR, - `cache/${label}-${products.join('-')}-runs-${formattedFrom}.json`); - try { - runs = JSON.parse(await fs.promises.readFile(cacheFile)); - if (runs.length) { - cachedCount++; - } - } catch (e) { - const url = `${runsUri}&from=${formattedFrom}&to=${formattedTo}`; - const response = await fetch(url); - // Many days do not have an aligned set of runs, but we always write to - // the cache to speed up future executions of this code. - runs = await response.json(); - await fs.promises.writeFile(cacheFile, JSON.stringify(runs)); - } - - if (!runs.length) { - continue; - } - - if (runs.length !== products.length) { - throw new Error( - `Fetched ${runs.length} runs, expected ${products.length}`); - } - - alignedRuns.set(formattedFrom, runs); - } - const after = moment(); - console.log(`Fetched ${alignedRuns.size} sets of runs in ` + - `${after - before} ms (${cachedCount} cached)`); - - return alignedRuns; -} // Score a set of runs (independently) on a set of tests. The runs are presumed // to be aligned in some way (i.e. they were all run at the same WPT SHA). @@ -276,7 +210,7 @@ async function main() { const from = moment(flags.get('from')); const to = moment(flags.get('to')); const experimental = flags.get('experimental'); - const alignedRuns = await fetchAlignedRunsFromServer( + const alignedRuns = await lib.runs.fetchAlignedRunsFromServer( products, from, to, experimental); // Verify that we have data for the fetched runs in the results-analysis-cache diff --git a/lib/runs.js b/lib/runs.js index 47c903ea1..737212254 100644 --- a/lib/runs.js +++ b/lib/runs.js @@ -1,6 +1,10 @@ 'use strict'; const fetch = require('node-fetch'); +const fs = require('fs'); +const moment = require('moment'); +const path = require('path'); +const {advanceDateToSkipBadDataIfNecessary} = require('../bad-ranges'); const RUNS_API = 'https://wpt.fyi/api/runs'; @@ -17,7 +21,7 @@ function apiURL(options = {}) { async function get(options) { const url = apiURL(options); - //console.log(`Fetching ${url}`); + // console.log(`Fetching ${url}`); return fetch(url).then(r => r.json()); } @@ -37,7 +41,7 @@ async function getAll(options) { } async function* getIterator(options) { - options = Object.assign({ 'max-count': 500 }, options); + options = Object.assign({'max-count': 500}, options); let url = apiURL(options); let previousUrl = null; @@ -51,7 +55,7 @@ async function* getIterator(options) { throw new Error(msg); } - let runs = await r.json(); + const runs = await r.json(); for (const run of runs) { yield run; } @@ -64,4 +68,73 @@ async function* getIterator(options) { } } -module.exports = { get, getAll, getIterator }; + +// Fetches aligned runs from the wpt.fyi server, between the |from| and |to| +// dates. If |experimental| is true fetch experimental runs, else stable runs. +// Returns a map of date to list of runs for that date (one per product) +// +// TODO: Known problem: there are periods of time, mostly mid-late 2018, where +// we ran both Safari 11.1 and 12.1, and the results are massively different. +// We should fetch multiple runs for each browser and have upgrade logic. +async function fetchAlignedRunsFromServer(products, from, to, experimental) { + const label = experimental ? 'experimental' : 'stable'; + let params = `label=master&label=${label}`; + for (const product of products) { + params += `&product=${product}`; + } + const runsUri = `${RUNS_API}?aligned=true&max-count=1&${params}`; + + console.log(`Fetching aligned runs from ${from.format('YYYY-MM-DD')} ` + + `to ${to.format('YYYY-MM-DD')}`); + + let cachedCount = 0; + const before = moment(); + const alignedRuns = new Map(); + while (from < to) { + const formattedFrom = from.format('YYYY-MM-DD'); + from.add(1, 'days'); + const formattedTo = from.format('YYYY-MM-DD'); + + // We advance the date (if necessary) before doing anything more, so that + // code later in the loop body can just 'continue' without checking. + from = advanceDateToSkipBadDataIfNecessary(from, experimental); + + // Attempt to read the runs from the cache. + // TODO: Consider https://github.com/tidoust/fetch-filecache-for-crawling + let runs; + const cacheFile = path.join(path.join(__dirname, '..'), + `cache/${label}-${products.join('-')}-runs-${formattedFrom}.json`); + try { + runs = JSON.parse(await fs.promises.readFile(cacheFile)); + if (runs.length) { + cachedCount++; + } + } catch (e) { + // No cache hit; load from the server instead. + const url = `${runsUri}&from=${formattedFrom}&to=${formattedTo}`; + const response = await fetch(url); + // Many days do not have an aligned set of runs, but we always write to + // the cache to speed up future executions of this code. + runs = await response.json(); + await fs.promises.writeFile(cacheFile, JSON.stringify(runs)); + } + + if (!runs.length) { + continue; + } + + if (runs.length !== products.length) { + throw new Error( + `Fetched ${runs.length} runs, expected ${products.length}`); + } + + alignedRuns.set(formattedFrom, runs); + } + const after = moment(); + console.log(`Fetched ${alignedRuns.size} sets of runs in ` + + `${after - before} ms (${cachedCount} cached)`); + + return alignedRuns; +} + +module.exports = {get, getAll, getIterator, fetchAlignedRunsFromServer}; diff --git a/package.json b/package.json index b84caa49c..91b43ed29 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ }, "scripts": { "install": "git clone --bare https://github.com/web-platform-tests/results-analysis-cache.git && git --git-dir=results-analysis-cache.git remote set-url --push origin git@github.com:web-platform-tests/results-analysis-cache.git || git --git-dir=results-analysis-cache.git fetch --tags", - "lint": "eslint browser-specific-failures.js lib/browser-specific.js compat-2021/main.js interop-2022/main.js", + "lint": "eslint browser-specific-failures.js lib/browser-specific.js lib/runs.js compat-2021/main.js interop-2022/main.js", "test": "npm run lint && npm run test-unit", "test-unit": "mocha -u bdd ./test/*.js" },