Fetch web api descriptions from MDN

README.md

@@ -74,6 +74,7 @@ A "Living Standard" ([example](https://xhr.spec.whatwg.org/)) should be added he
 
 - `browser.webidl.preprocessed.json`: a JSON file generated by Microsoft Edge. **Do not edit this file**.
     - Due to the different update schedules between Edge and TypeScript, this may not be the most up-to-date version of the spec.
+- `mdn/apiDescriptions.json`: a JSON file generated by fetching API descriptions from [MDN](https://developer.mozilla.org/en-US/docs/Web/API). **Do not edit this file**.
 - `addedTypes.json`: types that should exist in either browser or webworker but are missing from the Edge spec. The format of the file mimics that of `browser.webidl.preprocessed.json`.
 - `overridingTypes.json`: types that are defined in the spec file but has a better or more up-to-date definitions in the json files.
 - `removedTypes.json`: types that are defined in the spec file but should be removed.

inputfiles/README.md

@@ -4,6 +4,8 @@ Some files in this directory are generated.
 Please do not edit them.
 This specifically includes:
 
-* `browser.webidl.json`
+* `browser.webidl.preprocessed.json`
+* `idl/*`
+* `mdn/*`
 
 Feel free to send a pull request with changes to any other files.

package.json

@@ -3,7 +3,9 @@
   "private": true,
   "scripts": {
     "build": "tsc --p ./tsconfig.json && node ./lib/index.js",
-    "fetch": "tsc --p ./tsconfig.json && node ./lib/fetcher.js",
+    "fetch-idl": "npm run build && node ./lib/idlfetcher.js",
+    "fetch-mdn": "npm run build && node ./lib/mdnfetcher.js",
+    "fetch": "echo This could take a few minutes... && npm run fetch-idl && npm run fetch-mdn",
     "baseline-accept": "cpx \"generated\\*\" baselines\\",
     "test": "tsc --p ./tsconfig.json && node ./lib/index.js && node ./lib/test.js"
   },

src/emitter.ts

@@ -831,6 +831,10 @@ export function emitWebIDl(webidl: Browser.WebIdl, flavor: Flavor) {
             printer.printLineToStack(`interface ${processInterfaceType(i, i.name)} extends ${processedIName} {`);
         }
 
+        if (i.comment) {
+            printer.printLine(`/** ${i.comment} */`);
+        }
+
         printer.print(`interface ${processInterfaceType(i, processedIName)}`);
 
         const finalExtends = distinct([i.extends || "Object"].concat(i.implements || [])

src/index.ts

@@ -43,6 +43,7 @@ function emitDom() {
     const overriddenItems = require(path.join(inputFolder, "overridingTypes.json"));
     const addedItems = require(path.join(inputFolder, "addedTypes.json"));
     const comments = require(path.join(inputFolder, "comments.json"));
+    const documentationFromMDN = apiDescriptionsToIdl(require(path.join(inputFolder, 'mdn', 'apiDescriptions.json')));
     const removedItems = require(path.join(inputFolder, "removedTypes.json"));
     const idlSources: any[] = require(path.join(inputFolder, "idlSources.json"));
     const widlStandardTypes = idlSources.map(convertWidl);
@@ -60,6 +61,22 @@ function emitDom() {
         return result;
     }
 
+    function apiDescriptionsToIdl(descriptions: Record<string, string>) {
+        const idl: Browser.WebIdl = {
+            interfaces: {
+                interface: {}
+            }
+        };
+
+        Object.keys(descriptions).forEach(name => {
+            idl.interfaces!.interface[name] = {
+                comment: descriptions[name],
+            } as Browser.Interface;
+        });
+
+        return idl;
+    }
+
     /// Load the input file
     let webidl: Browser.WebIdl = require(path.join(inputFolder, "browser.webidl.preprocessed.json"));
 
@@ -106,6 +123,8 @@ function emitDom() {
             }
         }
     }
+
+    webidl = merge(webidl, documentationFromMDN);
     webidl = prune(webidl, removedItems);
     webidl = merge(webidl, addedItems);
     webidl = merge(webidl, overriddenItems);

src/mdnfetcher.ts

@@ -0,0 +1,44 @@
+import * as fs from "fs";
+import fetch from "node-fetch";
+import { JSDOM } from "jsdom";
+
+interface Page {
+    summary: string;
+}
+
+const knownBadDescriptions = [
+    "CustomEvent", // too vague; references truncated data
+    "IDBTransaction", // just a comment about Firefox 40
+    "RTCDtmfSender", // is just 'tbd'
+    "SVGMatrix", // too vague; references truncated data
+];
+
+fetchInterfaceDescriptions();
+
+async function fetchInterfaceDescriptions() {
+    const webIdl = require("../inputfiles/browser.webidl.preprocessed.json");
+    const interfaceNames = Object.keys(webIdl.interfaces.interface).sort();
+    const descriptions: Record<string, string> = {};
+
+    await interfaceNames.reduce(async (previousRequest, name) => {
+        // Issuing too many requests in parallel causes 504 gateway errors, so chain
+        await previousRequest;
+
+        const response = await fetch(`https://developer.mozilla.org/en-US/docs/Web/API/${name}$json`);
+        if (response.ok) {
+            const page = await response.json();
+            addDescription(name, page);
+        } else if (response.status !== 404) {
+            throw new Error(`Failed to fetch ${name}: ${response.statusText}`);
+        }
+    }, Promise.resolve());
+
+    function addDescription(name: string, page: Page) {
+        if (page.summary && !knownBadDescriptions.includes(name)) {
+            const fragment = JSDOM.fragment(page.summary);
+            descriptions[name] = fragment.textContent!;
+        }
+    }
+
+    fs.writeFileSync("inputfiles/mdn/apiDescriptions.json", JSON.stringify(descriptions, null, 2));
+}

src/types.d.ts

@@ -149,6 +149,7 @@ export interface TypeParameter {
 export interface Interface {
     name: string;
     extends: string;
+    comment?: string;
     constants?: {
         constant: Record<string, Constant>;
     }