Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

initial refactor of enriched document into speech document #1217

Closed
wants to merge 1 commit into from
Closed

initial refactor of enriched document into speech document #1217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions components/mjs/a11y/speech/config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"build": {
"component": "a11y/speech",
"targets": ["a11y/speech.ts", "a11y/speech"]
},
"webpack": {
"name": "a11y/speech",
"libs": [
"components/src/a11y/semantic-enrich/lib",
"components/src/a11y/sre/lib",
"components/src/input/mml/lib",
"components/src/core/lib"
]
}
}
8 changes: 8 additions & 0 deletions components/mjs/a11y/speech/speech.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import './lib/speech.js';

import {SpeechHandler} from '#js/a11y/speech.js';
import {hasWindow} from '#js/util/context.js';

if (MathJax.startup && hasWindow) {
MathJax.startup.extendHandler(handler => SpeechHandler(handler));
}
14 changes: 5 additions & 9 deletions ts/a11y/explorer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,7 @@ import { Handler } from '../core/Handler.js';
import { MmlNode } from '../core/MmlTree/MmlNode.js';
import { MathML } from '../input/mathml.js';
import { STATE, newState } from '../core/MathItem.js';
import {
EnrichedMathItem,
EnrichedMathDocument,
EnrichHandler,
} from './semantic-enrich.js';
import { SpeechMathItem, SpeechMathDocument, SpeechHandler } from './speech.js';
import { MathDocumentConstructor } from '../core/MathDocument.js';
import { OptionList, expandable } from '../util/Options.js';
import { SerializedMmlVisitor } from '../core/MmlTree/SerializedMmlVisitor.js';
Expand All @@ -48,8 +44,8 @@ export type Constructor<T> = new (...args: any[]) => T;
* Shorthands for types with HTMLElement, Text, and Document instead of generics
*/
export type HANDLER = Handler<HTMLElement, Text, Document>;
export type HTMLDOCUMENT = EnrichedMathDocument<HTMLElement, Text, Document>;
export type HTMLMATHITEM = EnrichedMathItem<HTMLElement, Text, Document>;
export type HTMLDOCUMENT = SpeechMathDocument<HTMLElement, Text, Document>;
export type HTMLMATHITEM = SpeechMathItem<HTMLElement, Text, Document>;
export type MATHML = MathML<HTMLElement, Text, Document>;

/*==========================================================================*/
Expand Down Expand Up @@ -290,8 +286,8 @@ export function ExplorerHandler(
handler: HANDLER,
MmlJax: MATHML = null
): HANDLER {
if (!handler.documentClass.prototype.enrich && MmlJax) {
handler = EnrichHandler(handler, MmlJax);
if (!handler.documentClass.prototype.attachSpeech) {
handler = SpeechHandler(handler, MmlJax);
}
handler.documentClass = ExplorerMathDocumentMixin(
handler.documentClass as any
Expand Down
193 changes: 0 additions & 193 deletions ts/a11y/semantic-enrich.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ import { SerializedMmlVisitor } from '../core/MmlTree/SerializedMmlVisitor.js';
import { OptionList, expandable } from '../util/Options.js';
import * as Sre from './sre.js';
import { GeneratorPool } from './speech/GeneratorPool.js';
import { WorkerHandler } from './speech/WebWorker.js';

/*==========================================================================*/

Expand All @@ -56,11 +55,6 @@ export type Constructor<T> = new (...args: any[]) => T;
*/
newState('ENRICHED', STATE.COMPILED + 10);

/**
* Add STATE value for adding speech (after INSERTED)
*/
newState('ATTACHSPEECH', STATE.INSERTED + 10);

/*==========================================================================*/

export class enrichVisitor<N, T, D> extends SerializedMmlVisitor {
Expand Down Expand Up @@ -125,11 +119,6 @@ export interface EnrichedMathItem<N, T, D> extends MathItem<N, T, D> {
*/
enrich(document: MathDocument<N, T, D>, force?: boolean): void;

/**
* @param {MathDocument} document The document where enrichment is occurring
*/
attachSpeech(document: MathDocument<N, T, D>): void;

/**
* @param {MathDocument} document The MathDocument for the MathItem
*/
Expand Down Expand Up @@ -256,30 +245,6 @@ export function EnrichedMathItemMixin<
`${prefix}"${maction[id].attributes.get('selection')}"`
);
}

/**
* Attaches the aria labels for speech and braille.
*
* @param {MathDocument} document The MathDocument for the MathItem
*/
public attachSpeech(document: MathDocument<N, T, D>) {
if (this.state() >= STATE.ATTACHSPEECH) return;
this.state(STATE.ATTACHSPEECH);
if (this.isEscaped || !document.options.enableEnrichment) return;
this.generatorPool.init(
document.options,
document.adaptor,
(document as EnrichedMathDocument<N, T, D>).webworker
);
if (document.options.enableSpeech || document.options.enableBraille) {
try {
this.outputData.mml = this.toMathML(this.root, this);
this.generatorPool.Speech(this.typesetRoot, this.outputData.mml);
} catch (err) {
document.options.speechError(document, this, err);
}
}
}
};
}

Expand All @@ -301,13 +266,6 @@ export interface EnrichedMathDocument<N, T, D>
*/
enrich(): EnrichedMathDocument<N, T, D>;

/**
* Attach speech to the MathItems in the MathDocument
*
* @returns {EnrichedMathDocument} The MathDocument (so calls can be chained)
*/
attachSpeech(): EnrichedMathDocument<N, T, D>;

/**
* @param {EnrichedMathDocument} doc The MathDocument for the error
* @param {EnrichedMathItem} math The MathItem causing the error
Expand All @@ -318,19 +276,6 @@ export interface EnrichedMathDocument<N, T, D>
math: EnrichedMathItem<N, T, D>,
err: Error
): void;

/**
* @param {EnrichedMathDocument} doc The MathDocument for the error
* @param {EnrichedMathItem} math The MathItem causing the error
* @param {Error} err The error being processed
*/
speechError(
doc: EnrichedMathDocument<N, T, D>,
math: EnrichedMathItem<N, T, D>,
err: Error
): void;

webworker: WorkerHandler<N, T, D>;
}

/**
Expand Down Expand Up @@ -368,31 +313,10 @@ export function EnrichedMathDocumentMixin<
math: EnrichedMathItem<N, T, D>,
err: Error
) => doc.enrichError(doc, math, err),
speechError: (
doc: EnrichedMathDocument<N, T, D>,
math: EnrichedMathItem<N, T, D>,
err: Error
) => doc.speechError(doc, math, err),
renderActions: expandable({
...BaseDocument.OPTIONS.renderActions,
enrich: [STATE.ENRICHED],
attachSpeech: [STATE.ATTACHSPEECH],
}),
worker: {
path: 'https://localhost',
basedir: 'sre',
pool: 'speech-workerpool.html',
worker: 'speech-worker.js',
sre: 'sre.js',
debug: false,
},
/* prettier-ignore */
speechTiming: {
asynchronous: true, // true to allow screen updates while adding speech, false to not
initial: 100, // initial delay until starting to add speech
threshold: 250, // time (in milliseconds) to process speech before letting screen update
intermediate: 10 // delay after processing speech reaches the threshold
},
/* prettier-ignore */
sre: expandable({
speech: 'none', // by default no speech is included
Expand All @@ -410,23 +334,6 @@ export function EnrichedMathDocumentMixin<
}),
};

/**
* The list of MathItems that need to be processed for speech
*/
protected awaitingSpeech: MathItem<N, T, D>[];

/**
* The identifier from setTimeout for the next speech loop
*/
protected speechTimeout: number = 0;

/**
* The function to resolve when the speech loop finishes
*/
protected attachSpeechDone: () => void;

public webworker: WorkerHandler<N, T, D> = null;

/**
* Enrich the MathItem class used for this MathDocument, and create the
* temporary MathItem used for enrchment
Expand Down Expand Up @@ -454,92 +361,6 @@ export function EnrichedMathDocumentMixin<
>(this.options.MathItem, MmlJax, toMathML);
}

// TODO: Do we still need async handling here?
/**
* Attach speech from a MathItem to a node
*
* @returns {EnrichedMathDocument} The object for chaining.
*/
public attachSpeech(): EnrichedMathDocument<N, T, D> {
if (!this.processed.isSet('attach-speech')) {
if (this.options.enableSpeech || this.options.enableBraille) {
if (!this.webworker) {
this.webworker = new WorkerHandler(
this.adaptor,
this.options.worker
);
this.webworker.Start();
this.webworker.Import();
}
if (this.options.speechTiming.asynchronous) {
this.attachSpeechAsync();
} else {
this.attachSpeechSync();
}
}
this.processed.set('attach-speech');
}
return this;
}

/**
* Add speech synchronously (e.g., for use in node applications on the server)
*/
protected attachSpeechSync() {
for (const math of this.math) {
(math as EnrichedMathItem<N, T, D>).attachSpeech(this);
}
}

/**
* Add speech in small chunks, allowing screen updates in between
* (e.g., helpful with lazy typesetting)
*/
protected attachSpeechAsync() {
if (this.speechTimeout) {
clearTimeout(this.speechTimeout);
this.speechTimeout = 0;
this.attachSpeechDone();
}
this.awaitingSpeech = Array.from(this.math);
if (this.awaitingSpeech.length === 0) {
this.awaitingSpeech = null;
return;
}
this.renderPromises.push(
new Promise<void>((ok, _fail) => {
this.attachSpeechDone = ok;
})
);
this.speechTimeout = setTimeout(
() => this.attachSpeechLoop(),
this.options.speechTiming.initial
);
}

/**
* Loops through math items to attach speech until the timeout threshold is reached.
*/
protected attachSpeechLoop() {
const timing = this.options.speechTiming;
const awaitingSpeech = this.awaitingSpeech;
const timeStart = new Date().getTime();
const timeEnd = timeStart + timing.threshold;
do {
const math = awaitingSpeech.shift();
(math as EnrichedMathItem<N, T, D>).attachSpeech(this);
} while (awaitingSpeech.length && new Date().getTime() < timeEnd);
if (awaitingSpeech.length) {
this.speechTimeout = setTimeout(
() => this.attachSpeechLoop(),
timing.intermediate
);
} else {
this.speechTimeout = 0;
this.attachSpeechDone();
}
}

/**
* Enrich the MathItems in this MathDocument
*
Expand Down Expand Up @@ -569,17 +390,6 @@ export function EnrichedMathDocumentMixin<
console.warn('Enrichment error:', err);
}

/**
* @override
*/
public speechError(
_doc: EnrichedMathDocument<N, T, D>,
_math: EnrichedMathItem<N, T, D>,
err: Error
) {
console.warn('Speech generation error:', err);
}

/**
* @override
*/
Expand All @@ -593,9 +403,6 @@ export function EnrichedMathDocumentMixin<
}
}
}
if (state < STATE.ATTACHSPEECH) {
this.processed.clear('attach-speech');
}
return this;
}
};
Expand Down
Loading