axe takes too long on page, causes protocol timeout

[james-muriithi]

Error: Error: Lighthouse returned error: INTERNAL: Extension result empty. Lighthouse calling Page.navigate with https://james-muriithi.github.io/jCheckBox/ Uncaught (in promise) Error: Extension timed out. No call to exit() before render timeout. at wrs.Extension.onTimeout_ (http://ghost-rider/devtools_script_api.js:5555:13) at http://ghost-rider/devtools_script_api.js:708:9 at Map.forEach () at chromium.DevTools.Connection.dispatchMessage (http://ghost-rider/devtools_script_api.js:707:76) at chromium.DevTools.Connection.onJsonMessage_ (http://ghost-rider/devtools_script_api.js:689:65) at :1:40

[robdodson]

I was able to confirm that both web.dev/measure and running Lighthouse in devtools will time out on this page. Not sure sure. Transferring to the Lighthouse team.

[connorjclark]

When you say "time out" does that mean no results or partial? I just ran it twice and it finished (but with protocol timeout error from Runtime.Evaluate)

[connorjclark]

I get this timeout from CLI too. Times out on Accessibility gatherer driver.evaluateAsync. axe is taking too long.

[connorjclark]

color-contrast takes 231s on the given page. I'll look into why tomorrow.

async function test() {
  const rules = [
    "accesskeys",
    "aria-allowed-attr",
    "aria-hidden-body",
    "aria-hidden-focus",
    "aria-input-field-name",
    "aria-required-attr",
    "aria-required-children",
    "aria-required-parent",
    "aria-roles",
    "aria-toggle-field-name",
    "aria-valid-attr-value",
    "aria-valid-attr",
    "audio-caption",
    "button-name",
    "bypass",
    "color-contrast",
    "definition-list",
    "dlitem",
    "document-title",
    "duplicate-id-active",
    "duplicate-id-aria",
    "form-field-multiple-labels",
    "frame-title",
    "heading-order",
    "html-has-lang",
    "html-lang-valid",
    "image-alt",
    "input-button-name",
    "input-image-alt",
    "label",
    "layout-table",
    "link-name",
    "list",
    "listitem",
    "meta-refresh",
    "meta-viewport",
    "object-alt",
    "role-img-alt",
    "scrollable-region-focusable",
    "tabindex",
    "td-headers-attr",
    "th-has-data-cells",
    "valid-lang",
    "video-caption",
    "video-description"
  ];

  for (const rule of rules) {
    const start = performance.now();
    await window.axe.run(document, {
      elementRef: true,
      runOnly: {
        type: 'rule',
        values: [
          rule,
        ],
      },
      resultTypes: ['violations', 'inapplicable'],
      rules: {
        'tabindex': { enabled: true },
        'accesskeys': { enabled: true },
        'heading-order': { enabled: true },
        'duplicate-id': { enabled: false },
        'table-fake-caption': { enabled: false },
        'td-has-header': { enabled: false },
        'marquee': { enabled: false },
        'area-alt': { enabled: false },
        'aria-dpub-role-fallback': { enabled: false },
        'html-xml-lang-mismatch': { enabled: false },
        'blink': { enabled: false },
        'server-side-image-map': { enabled: false },
      },
      // @ts-ignore
    });
    const end = performance.now();
    console.log(end - start, rule);
  }

  console.log('done');
}

test();

603.1250000014552 "accesskeys"
571.8550000019604 "aria-allowed-attr"
563.5650000040187 "aria-hidden-body"
558.8050000005751 "aria-hidden-focus"
563.2450000048266 "aria-input-field-name"
544.5450000042911 "aria-required-attr"
542.6050000023679 "aria-required-children"
545.8250000010594 "aria-required-parent"
543.1899999966845 "aria-roles"
569.019999995362 "aria-toggle-field-name"
545.7099999985076 "aria-valid-attr-value"
547.9100000011385 "aria-valid-attr"
548.3800000001793 "audio-caption"
547.4449999965145 "button-name"
543.3349999948405 "bypass"
231475.1150000011 "color-contrast"
1809.4550000023446 "definition-list"
1999.390000004496 "dlitem"
2000.3500000020722 "document-title"
1999.374999999418 "duplicate-id-active"
1999.7549999970943 "duplicate-id-aria"
1999.3049999975483 "form-field-multiple-labels"
2000.5799999998999 "frame-title"
1999.8899999991409 "heading-order"
1999.4500000029802 "html-has-lang"
1999.4799999985844 "html-lang-valid"
1999.100000000908 "image-alt"
1999.69499999861 "input-button-name"
1999.7250000014901 "input-image-alt"
1999.8450000057346 "label"
1999.5600000038394 "layout-table"
2000.7499999992433 "link-name"
1999.1300000037882 "list"
1999.7550000043702 "listitem"
1999.195000003965 "meta-refresh"
2000.3900000010617 "meta-viewport"
1999.164999993809 "object-alt"
1999.5749999943655 "role-img-alt"
1300.1000000003842 "scrollable-region-focusable"
546.2550000011106 "tabindex"
543.1900000039604 "td-headers-attr"
1609.5100000020466 "th-has-data-cells"
1999.4999999980791 "valid-lang"
1999.7250000014901 "video-caption"
1999.709999996412 "video-description"

[connorjclark]

Filed an issue with axe. My first impression is that this will be unfixable. We could split our axe gatherer into two runs: color-contrast (and any other possibly slow ones) and all the rest. That would reduce the failure scope to just the one accessibility audit, instead of all of them.

EDIT: spoke way too soon. There's a fix coming that would reduce it to at least 10s, which is pretty good.

[paulirish]

We could split our axe gatherer into two runs: color-contrast (and any other possibly slow ones) and all the rest.

related: #9622 "Entire accessibility gatherer runs via SEO category"

[paulirish]

The fix landed and shipped in axe v3.5.0

here's our PR to update to 3.5.1 ? #10344

[connorjclark]

verified fix.