core: create flag to prevent fatal error on bad status code

cli/cli-flags.js

@@ -204,12 +204,16 @@ function getYargsParser(manualArgv) {
         type: 'boolean',
         describe: 'Disables collection of the full page screenshot, which can be quite large',
       },
+      'ignore-status-code': {
+        type: 'boolean',
+        describe: 'Disables failing on 404 status code, and instead issues a warning.',
+      },
     })
     .group([
       'save-assets', 'list-all-audits', 'list-locales', 'list-trace-categories', 'additional-trace-categories',
       'config-path', 'preset', 'chrome-flags', 'port', 'hostname', 'form-factor', 'screenEmulation', 'emulatedUserAgent',
       'max-wait-for-load', 'enable-error-reporting', 'gather-mode', 'audit-mode',
-      'only-audits', 'only-categories', 'skip-audits', 'budget-path', 'disable-full-page-screenshot',
+      'only-audits', 'only-categories', 'skip-audits', 'budget-path', 'disable-full-page-screenshot', 'ignore-status-code',
     ], 'Configuration:')
 
     // Output

core/config/constants.js

@@ -113,6 +113,7 @@ const defaultSettings = {
   disableFullPageScreenshot: false,
   skipAboutBlank: false,
   blankPage: 'about:blank',
+  ignoreStatusCode: false,
 
   // the following settings have no defaults but we still want ensure that `key in settings`
   // in config will work in a typechecked way

core/gather/navigation-runner.js

@@ -175,6 +175,7 @@ async function _computeNavigationResult(
     ? getPageLoadError(navigationError, {
       url: mainDocumentUrl,
       loadFailureMode: navigationContext.navigation.loadFailureMode,
+      ignoreStatusCode: navigationContext.resolvedConfig.settings.ignoreStatusCode,
       networkRecords: debugData.records,
       warnings,
     })

core/lib/navigation-error.js

@@ -16,6 +16,13 @@ const UIStrings = {
    */
   warningXhtml:
     'The page MIME type is XHTML: Lighthouse does not explicitly support this document type',
+  /**
+   * Warning shown in report when the page under test responds with a 404, which Lighthouse is not able to reliably load
+   * so we display a warning.
+   */
+  warning404: 'Lighthouse was unable to reliably load the page you requested. Make sure' +
+    ' you are testing the correct URL and that the server is properly responding' +
+    ' to all requests. (Status code: 404)',
 };
 
 const str_ = i18n.createIcuMessageFn(import.meta.url, UIStrings);
@@ -27,9 +34,10 @@ const XHTML_MIME_TYPE = 'application/xhtml+xml';
 /**
  * Returns an error if the original network request failed or wasn't found.
  * @param {LH.Artifacts.NetworkRequest|undefined} mainRecord
+ * @param {{warnings: Array<string | LH.IcuMessage>, ignoreStatusCode?: LH.Config.Settings['ignoreStatusCode']}} context
  * @return {LH.LighthouseError|undefined}
  */
-function getNetworkError(mainRecord) {
+function getNetworkError(mainRecord, context) {
   if (!mainRecord) {
     return new LighthouseError(LighthouseError.errors.NO_DOCUMENT_REQUEST);
   } else if (mainRecord.failed) {
@@ -46,6 +54,8 @@ function getNetworkError(mainRecord) {
       return new LighthouseError(
         LighthouseError.errors.FAILED_DOCUMENT_REQUEST, {errorDetails: netErr});
     }
+  } else if (mainRecord.statusCode === 404 && context.ignoreStatusCode) {
+    context.warnings.push(str_(UIStrings.warning404));
   } else if (mainRecord.hasErrorStatusCode()) {
     return new LighthouseError(LighthouseError.errors.ERRORED_DOCUMENT_REQUEST, {
       statusCode: `${mainRecord.statusCode}`,
@@ -113,7 +123,7 @@ function getNonHtmlError(finalRecord) {
  * Returns an error if the page load should be considered failed, e.g. from a
  * main document request failure, a security issue, etc.
  * @param {LH.LighthouseError|undefined} navigationError
- * @param {{url: string, loadFailureMode: LH.Config.SharedPassNavigationJson['loadFailureMode'], networkRecords: Array<LH.Artifacts.NetworkRequest>, warnings: Array<string | LH.IcuMessage>}} context
+ * @param {{url: string, loadFailureMode: LH.Config.SharedPassNavigationJson['loadFailureMode'], ignoreStatusCode?: LH.Config.Settings['ignoreStatusCode'], networkRecords: Array<LH.Artifacts.NetworkRequest>, warnings: Array<string | LH.IcuMessage>}} context
  * @return {LH.LighthouseError|undefined}
  */
 function getPageLoadError(navigationError, context) {
@@ -144,7 +154,7 @@ function getPageLoadError(navigationError, context) {
     context.warnings.push(str_(UIStrings.warningXhtml));
   }
 
-  const networkError = getNetworkError(mainRecord);
+  const networkError = getNetworkError(mainRecord, context);
   const interstitialError = getInterstitialError(mainRecord, networkRecords);
   const nonHtmlError = getNonHtmlError(finalRecord);
 

core/test/lib/navigation-error-test.js

@@ -31,9 +31,10 @@ function makeNetworkRequest() {
 describe('#getNetworkError', () => {
   /**
    * @param {NetworkRequest=} mainRecord
+   * @param {{warnings: Array<string | LH.IcuMessage>, ignoreStatusCode?: LH.Config.Settings['ignoreStatusCode']}=} context
    */
-  function getAndExpectError(mainRecord) {
-    const error = getNetworkError(mainRecord);
+  function getAndExpectError(mainRecord, context) {
+    const error = getNetworkError(mainRecord, {warnings: [], ...context});
     if (!error) throw new Error('expected a network error');
     return error;
   }
@@ -42,7 +43,7 @@ describe('#getNetworkError', () => {
     const url = 'http://the-page.com';
     const mainRecord = makeNetworkRequest();
     mainRecord.url = url;
-    expect(getNetworkError(mainRecord)).toBeUndefined();
+    expect(getNetworkError(mainRecord, {warnings: []})).toBeUndefined();
   });
 
   it('fails when page fails to load', () => {
@@ -66,15 +67,41 @@ describe('#getNetworkError', () => {
     expect(error.friendlyMessage).toBeDisplayString(/^Lighthouse was unable to reliably load/);
   });
 
-  it('fails when page returns with a 404', () => {
+  it('warns when page returns with a 404 with flag', () => {
     const url = 'http://the-page.com';
     const mainRecord = makeNetworkRequest();
     mainRecord.url = url;
     mainRecord.statusCode = 404;
-    const error = getAndExpectError(mainRecord);
+    const context = {
+      url,
+      networkRecords: [mainRecord],
+      warnings: [],
+      loadFailureMode: LoadFailureMode.warn,
+      ignoreStatusCode: true,
+    };
+
+    const error = getNetworkError(mainRecord, context);
+    expect(error).toBeUndefined();
+    expect(context.warnings[0]).toBeDisplayString(/^Lighthouse was unable to reliably load/);
+  });
+
+  it('fails when page returns with a 404 without flag', () => {
+    const url = 'http://the-page.com';
+    const mainRecord = makeNetworkRequest();
+    mainRecord.url = url;
+    mainRecord.statusCode = 404;
+    const context = {
+      url,
+      networkRecords: [mainRecord],
+      warnings: [],
+      loadFailureMode: LoadFailureMode.warn,
+    };
+
+    const error = getAndExpectError(mainRecord, context);
+    expect(error).toBeDefined();
     expect(error.message).toEqual('ERRORED_DOCUMENT_REQUEST');
     expect(error.code).toEqual('ERRORED_DOCUMENT_REQUEST');
-    expect(error.friendlyMessage).toBeDisplayString(/^Lighthouse was unable to reliably load.*404/);
+    expect(error.friendlyMessage).toBeDisplayString(/^Lighthouse was unable to reliably load/);
   });
 
   it('fails when page returns with a 500', () => {

core/test/results/artifacts/artifacts.json

@@ -71,6 +71,7 @@
     "disableFullPageScreenshot": false,
     "skipAboutBlank": false,
     "blankPage": "about:blank",
+    "ignoreStatusCode": false,
     "budgets": [
       {
         "path": "/",

types/lhr/settings.d.ts

@@ -114,6 +114,9 @@ export type ScreenEmulationSettings = {
   blankPage?: string;
   /** Disables collection of the full page screenshot, which can be rather large and possibly leave the page in an undesirable state. */
   disableFullPageScreenshot?: boolean;
+
+  /** Disables failing on 404 status code, and instead issues a warning */
+  ignoreStatusCode?: boolean;
 }
 
 export interface ConfigSettings extends Required<SharedFlagsSettings> {