Highlight images during text-to-speech

readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsNavigator.kt

@@ -68,7 +68,7 @@ public class TtsNavigator<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  ?: ttsEngineProvider.createEmptyPreferences()
 
  val contentIterator =
- TtsContentIterator(publication, tokenizerFactory, initialLocator)
+ TtsUtteranceIterator(publication, tokenizerFactory, initialLocator)
 
  val ttsEngine =
  ttsEngineProvider.createEngine(publication, actualInitialPreferences)
@@ -279,28 +279,24 @@ public class TtsNavigator<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  private fun TtsPlayer.Utterance.toPosition(): Location {
  val currentLink = publication.readingOrder[position.resourceIndex]
 
- val utteranceHighlight = publication
+ val utteranceLocator = publication
  .locatorFromLink(currentLink)!!
  .copy(
  locations = position.locations,
- text = Locator.Text(
- highlight = text,
- before = position.textBefore,
- after = position.textAfter
- )
+ text = position.text
  )
 
- val tokenHighlight = range
- ?.let { utteranceHighlight.copy(text = utteranceHighlight.text.substring(it)) }
+ val tokenLocator = range
+ ?.let { utteranceLocator.copy(text = utteranceLocator.text.substring(it)) }
 
  return Location(
  href = Href(currentLink.href),
- textBefore = position.textBefore,
- textAfter = position.textAfter,
+ textBefore = position.text.before,
+ textAfter = position.text.after,
  utterance = text,
  range = range,
- utteranceLocator = utteranceHighlight,
- tokenLocator = tokenHighlight
+ utteranceLocator = utteranceLocator,
+ tokenLocator = tokenLocator
  )
  }
 }

readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsPlayer.kt

@@ -24,13 +24,13 @@ import org.readium.r2.shared.extensions.tryOrNull
 import org.readium.r2.shared.publication.Locator
 
 /**
- * Plays the content from a [TtsContentIterator] with a [TtsEngine].
+ * Plays the content from a [TtsUtteranceIterator] with a [TtsEngine].
  */
 @ExperimentalReadiumApi
 internal class TtsPlayer<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  E : TtsEngine.Error, V : TtsEngine.Voice> private constructor(
  private val engineFacade: TtsEngineFacade<S, P, E, V>,
- private val contentIterator: TtsContentIterator,
+ private val contentIterator: TtsUtteranceIterator,
  initialWindow: UtteranceWindow,
  initialPreferences: P
 ) : Configurable<S, P> {
@@ -40,7 +40,7 @@ internal class TtsPlayer<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  suspend operator fun <S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  E : TtsEngine.Error, V : TtsEngine.Voice> invoke(
  engine: TtsEngine<S, P, E, V>,
- contentIterator: TtsContentIterator,
+ contentIterator: TtsUtteranceIterator,
  initialPreferences: P
  ): TtsPlayer<S, P, E, V>? {
  val initialContext = tryOrNull { contentIterator.startContext() }
@@ -59,23 +59,23 @@ internal class TtsPlayer<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  )
  }
 
- private suspend fun TtsContentIterator.startContext(): UtteranceWindow? {
- val previousUtterance = previousUtterance()
- val currentUtterance = nextUtterance()
+ private suspend fun TtsUtteranceIterator.startContext(): UtteranceWindow? {
+ val previousUtterance = previous()
+ val currentUtterance = next()
 
  val startWindow = if (currentUtterance != null) {
  UtteranceWindow(
  previousUtterance = previousUtterance,
  currentUtterance = currentUtterance,
- nextUtterance = nextUtterance(),
+ nextUtterance = next(),
  ended = false
  )
  } else {
  val actualCurrentUtterance = previousUtterance ?: return null
- val actualPreviousUtterance = previousUtterance()
+ val actualPreviousUtterance = previous()
 
  // Go back to the end of the iterator.
- nextUtterance()
+ next()
 
  UtteranceWindow(
  previousUtterance = actualPreviousUtterance,
@@ -129,15 +129,14 @@ internal class TtsPlayer<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  data class Position(
  val resourceIndex: Int,
  val locations: Locator.Locations,
- val textBefore: String?,
- val textAfter: String?
+ val text: Locator.Text
  )
  }
 
  private data class UtteranceWindow(
- val previousUtterance: TtsContentIterator.Utterance?,
- val currentUtterance: TtsContentIterator.Utterance,
- val nextUtterance: TtsContentIterator.Utterance?,
+ val previousUtterance: TtsUtteranceIterator.Utterance?,
+ val currentUtterance: TtsUtteranceIterator.Utterance,
+ val nextUtterance: TtsUtteranceIterator.Utterance?,
  val ended: Boolean = false
  )
 
@@ -411,19 +410,19 @@ internal class TtsPlayer<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  val previousUtterance =
  try {
  // Get previously currentUtterance once more
- contentIterator.previousUtterance()
+ contentIterator.previous()
 
  // Get previously previousUtterance once more
- contentIterator.previousUtterance()
+ contentIterator.previous()
 
  // Get new previous utterance
- val previousUtterance = contentIterator.previousUtterance()
+ val previousUtterance = contentIterator.previous()
 
  // Go to currentUtterance position
- contentIterator.nextUtterance()
+ contentIterator.next()
 
  // Go to nextUtterance position
- contentIterator.nextUtterance()
+ contentIterator.next()
 
  previousUtterance
  } catch (e: Exception) {
@@ -448,7 +447,7 @@ internal class TtsPlayer<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  }
 
  val nextUtterance = try {
- contentIterator.nextUtterance()
+ contentIterator.next()
  } catch (e: Exception) {
  onContentError(e)
  return
@@ -498,8 +497,8 @@ internal class TtsPlayer<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  playContinuous()
  }
 
- private suspend fun speakUtterance(utterance: TtsContentIterator.Utterance): E? =
- engineFacade.speak(utterance.text, utterance.language, ::onRangeChanged)
+ private suspend fun speakUtterance(utterance: TtsUtteranceIterator.Utterance): E? =
+ engineFacade.speak(utterance.utterance, utterance.language, ::onRangeChanged)
 
  private fun onEngineError(error: E) {
  playbackMutable.value = playbackMutable.value.copy(
@@ -535,15 +534,14 @@ internal class TtsPlayer<S : TtsEngine.Settings, P : TtsEngine.Preferences<P>,
  private fun isPlaying() =
  playbackMutable.value.playWhenReady && playback.value.state == State.Ready
 
- private fun TtsContentIterator.Utterance.ttsPlayerUtterance(): Utterance =
+ private fun TtsUtteranceIterator.Utterance.ttsPlayerUtterance(): Utterance =
  Utterance(
- text = text,
+ text = utterance,
  range = null,
  position = Utterance.Position(
  resourceIndex = resourceIndex,
  locations = locations,
- textAfter = textAfter,
- textBefore = textBefore
+ text = text
  )
  )
 }

readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsContentIterator.kt → readium/navigator/src/main/java/org/readium/r2/navigator/media3/tts/TtsUtteranceIterator.kt

@@ -24,17 +24,16 @@ import org.readium.r2.shared.util.tokenizer.TextTokenizer
  *
  * Not thread-safe.
  */
-internal class TtsContentIterator(
+internal class TtsUtteranceIterator(
  private val publication: Publication,
  private val tokenizerFactory: (language: Language?) -> TextTokenizer,
  initialLocator: Locator?
 ) {
  data class Utterance(
+ val utterance: String,
  val resourceIndex: Int,
  val locations: Locator.Locations,
- val text: String,
- val textBefore: String?,
- val textAfter: String?,
+ val text: Locator.Text,
  val language: Language?
  )
 
@@ -109,14 +108,14 @@ internal class TtsContentIterator(
  /**
  * Advances to the previous item and returns it, or null if we reached the beginning.
  */
- suspend fun previousUtterance(): Utterance? =
- nextUtterance(Direction.Backward)
+ suspend fun previous(): Utterance? =
+ next(Direction.Backward)
 
  /**
  * Advances to the next item and returns it, or null if we reached the end.
  */
- suspend fun nextUtterance(): Utterance? =
- nextUtterance(Direction.Forward)
+ suspend fun next(): Utterance? =
+ next(Direction.Forward)
 
  private enum class Direction {
  Forward, Backward;
@@ -126,10 +125,10 @@ internal class TtsContentIterator(
  * Gets the next utterance in the given [direction], or null when reaching the beginning or the
  * end.
  */
- private suspend fun nextUtterance(direction: Direction): Utterance? {
+ private suspend fun next(direction: Direction): Utterance? {
  val utterance = utterances.nextIn(direction)
  if (utterance == null && loadNextUtterances(direction)) {
- return nextUtterance(direction)
+ return next(direction)
  }
  return utterance
  }
@@ -167,7 +166,7 @@ internal class TtsContentIterator(
  */
  private fun Content.Element.tokenize(): List<Content.Element> {
  val contentTokenizer = TextContentTokenizer(
- language = this@TtsContentIterator.language,
+ language = this@TtsUtteranceIterator.language,
  textTokenizerFactory = tokenizerFactory,
  overrideContentLanguage = overrideContentLanguage
  )
@@ -187,11 +186,10 @@ internal class TtsContentIterator(
  ?: throw IllegalStateException("Content Element cannot be found in readingOrder.")
 
  return Utterance(
+ utterance = text,
  resourceIndex = resourceIndex,
  locations = locator.locations,
- text = text,
- textBefore = locator.text.before,
- textAfter = locator.text.after,
+ text = locator.text,
  language = language
  )
  }