Add a TTS navigator with background playback (#333)

Author: qnga

.github/workflows/checks.yml

@@ -1,6 +1,7 @@
 name: Checks
 
 on:
+  workflow_dispatch:
   push:
     branches: [ main, develop ]
   pull_request:

gradle/libs.versions.toml

@@ -23,6 +23,7 @@ androidx-lifecycle = "2.5.1"
 androidx-lifecycle-extensions = "2.2.0"
 androidx-media = "1.6.0"
 androidx-media2 = "1.2.1"
+androidx-media3 = "1.0.0-rc01"
 androidx-navigation = "2.5.2"
 androidx-paging = "3.1.1"
 androidx-recyclerview = "1.2.1"
@@ -87,6 +88,9 @@ androidx-lifecycle-vmsavedstate = { group = "androidx.lifecycle", name = "lifecy
 androidx-media = { group = "androidx.media", name = "media", version.ref = "androidx-media" }
 androidx-media2-session = { group = "androidx.media2", name = "media2-session", version.ref = "androidx-media2" }
 androidx-media2-player = { group = "androidx.media2", name = "media2-player", version.ref = "androidx-media2" }
+androidx-media3-common = { group = "androidx.media3", name = "media3-common", version.ref = "androidx-media3" }
+androidx-media3-session = { group = "androidx.media3", name = "media3-session", version.ref = "androidx-media3" }
+androidx-media3-exoplayer = { group = "androidx.media3", name = "media3-exoplayer", version.ref = "androidx-media3" }
 androidx-navigation-fragment = { group = "androidx.navigation", name = "navigation-fragment-ktx", version.ref = "androidx-navigation" }
 androidx-navigation-ui = { group = "androidx.navigation", name = "navigation-ui-ktx", version.ref = "androidx-navigation" }
 androidx-paging = { group = "androidx.paging", name = "paging-runtime-ktx", version.ref = "androidx-paging" }
@@ -138,6 +142,7 @@ coroutines = ["kotlinx-coroutines-core", "kotlinx-coroutines-android"]
 exoplayer = ["google-exoplayer-core", "google-exoplayer-ui", "google-exoplayer-mediasession", "google-exoplayer-workmanager", "google-exoplayer-extension-media2"]
 lifecycle = ["androidx-lifecycle-common", "androidx-lifecycle-extensions", "androidx-lifecycle-livedata", "androidx-lifecycle-runtime", "androidx-lifecycle-viewmodel", "androidx-lifecycle-vmsavedstate", "androidx-lifecycle-viewmodel-compose"]
 media2 = ["androidx-media2-session", "androidx-media2-player"]
+media3 = ["androidx-media3-session", "androidx-media3-common", "androidx-media3-exoplayer"]
 navigation = ["androidx-navigation-fragment", "androidx-navigation-ui"]
 room = ["androidx-room-runtime", "androidx-room-ktx"]
 test-frameworks = ["junit", "androidx-ext-junit", "androidx-expresso-core", "robolectric", "kotlin-junit", "assertj", "kotlinx-coroutines-test"]

readium/navigator/build.gradle.kts

@@ -67,6 +67,7 @@ dependencies {
     implementation(libs.bundles.lifecycle)
     implementation(libs.androidx.recyclerview)
     implementation(libs.androidx.media)
+    implementation(libs.bundles.media3)
     implementation(libs.androidx.viewpager2)
     implementation(libs.androidx.webkit)
     // Needed to avoid a crash with API 31, see https://stackoverflow.com/a/69152986/1474476

readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataFactory.kt

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2022 Readium Foundation. All rights reserved.
+ * Use of this source code is governed by the BSD-style license
+ * available in the top-level LICENSE file of the project.
+ */
+
+package org.readium.r2.navigator.media3.api
+
+import androidx.media3.common.MediaMetadata
+import androidx.media3.common.MediaMetadata.PICTURE_TYPE_FRONT_COVER
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import org.readium.r2.shared.publication.Publication
+
+/**
+ * Builds media metadata using the given title, author and cover,
+ * and fall back on what is in the publication.
+ */
+@androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class)
+internal class DefaultMediaMetadataFactory(
+    private val publication: Publication,
+    title: String? = null,
+    author: String? = null,
+    cover: ByteArray? = null
+) : MediaMetadataFactory {
+
+    private val coroutineScope =
+        CoroutineScope(Dispatchers.Default)
+
+    private val title: String =
+        title ?: publication.metadata.title
+
+    private val authors: String? =
+        author ?: publication.metadata.authors
+            .firstOrNull { it.name.isNotBlank() }?.name
+
+    private val cover: Deferred<ByteArray?> = coroutineScope.async {
+        cover ?: publication.linkWithRel("cover")
+            ?.let { publication.get(it) }
+            ?.read()
+            ?.getOrNull()
+    }
+
+    override suspend fun publicationMetadata(): MediaMetadata {
+        val builder = MediaMetadata.Builder()
+            .setTitle(title)
+            .setTotalTrackCount(publication.readingOrder.size)
+
+        authors
+            ?.let { builder.setArtist(it) }
+
+        cover.await()
+            ?.let { builder.maybeSetArtworkData(it, PICTURE_TYPE_FRONT_COVER) }
+
+        return builder.build()
+    }
+
+    override suspend fun resourceMetadata(index: Int): MediaMetadata {
+        val builder = MediaMetadata.Builder()
+            .setTrackNumber(index)
+            .setTitle(title)
+
+        authors
+            ?.let { builder.setArtist(it) }
+
+        cover.await()
+            ?.let { builder.maybeSetArtworkData(it, PICTURE_TYPE_FRONT_COVER) }
+
+        return builder.build()
+    }
+}

readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/DefaultMediaMetadataProvider.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2022 Readium Foundation. All rights reserved.
+ * Use of this source code is governed by the BSD-style license
+ * available in the top-level LICENSE file of the project.
+ */
+
+package org.readium.r2.navigator.media3.api
+
+import org.readium.r2.shared.publication.Publication
+
+/**
+ * Builds a [MediaMetadataFactory] which will use the given title, author and cover,
+ * and fall back on what is in the publication.
+ */
+class DefaultMediaMetadataProvider(
+    private val title: String? = null,
+    private val author: String? = null,
+    private val cover: ByteArray? = null
+) : MediaMetadataProvider {
+
+    override fun createMetadataFactory(publication: Publication): MediaMetadataFactory {
+        return DefaultMediaMetadataFactory(publication, title, author, cover)
+    }
+}

readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataFactory.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2022 Readium Foundation. All rights reserved.
+ * Use of this source code is governed by the BSD-style license
+ * available in the top-level LICENSE file of the project.
+ */
+
+package org.readium.r2.navigator.media3.api
+
+import androidx.media3.common.MediaMetadata
+
+/**
+ * Factory for the [MediaMetadata] associated with the publication and its resources.
+ *
+ * The metadata are used for example in the media-style Android notification.
+ */
+interface MediaMetadataFactory {
+
+    /**
+     * Creates the [MediaMetadata] for the whole publication.
+     */
+    suspend fun publicationMetadata(): MediaMetadata
+
+    /**
+     * Creates the [MediaMetadata] for the reading order resource at the given [index].
+     */
+    suspend fun resourceMetadata(index: Int): MediaMetadata
+}

readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaMetadataProvider.kt

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2022 Readium Foundation. All rights reserved.
+ * Use of this source code is governed by the BSD-style license
+ * available in the top-level LICENSE file of the project.
+ */
+
+package org.readium.r2.navigator.media3.api
+
+import org.readium.r2.shared.publication.Publication
+
+/**
+ *  To be implemented to use a custom [MediaMetadataFactory].
+ */
+fun interface MediaMetadataProvider {
+
+    fun createMetadataFactory(publication: Publication): MediaMetadataFactory
+}

readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/MediaNavigator.kt

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2022 Readium Foundation. All rights reserved.
+ * Use of this source code is governed by the BSD-style license
+ * available in the top-level LICENSE file of the project.
+ */
+
+package org.readium.r2.navigator.media3.api
+
+import androidx.media3.common.Player
+import kotlinx.coroutines.flow.StateFlow
+import org.readium.r2.navigator.Navigator
+import org.readium.r2.shared.ExperimentalReadiumApi
+import org.readium.r2.shared.util.Closeable
+
+@ExperimentalReadiumApi
+interface MediaNavigator<P : MediaNavigator.Position> : Navigator, Closeable {
+
+    /**
+     *  Marker interface for the [position] flow.
+     */
+    interface Position
+
+    /**
+     * State of the player.
+     */
+    sealed interface State {
+
+        /**
+         * The navigator is ready to play.
+         */
+        interface Ready : State
+
+        /**
+         * The end of the media has been reached.
+         */
+        interface Ended : State
+
+        /**
+         * The navigator cannot play because the buffer is starved.
+         */
+        interface Buffering : State
+
+        /**
+         * The navigator cannot play because an error occurred.
+         */
+        interface Error : State
+    }
+
+    /**
+     * State of the playback.
+     *
+     * @param state The current state.
+     * @param playWhenReady If the navigator should play as soon as the state is Ready.
+     */
+    data class Playback(
+        val state: State,
+        val playWhenReady: Boolean
+    )
+
+    /**
+     * Indicates the current state of the playback.
+     */
+    val playback: StateFlow<Playback>
+
+    val position: StateFlow<P>
+
+    /**
+     * Resumes the playback at the current location.
+     */
+    fun play()
+
+    /**
+     * Pauses the playback.
+     */
+    fun pause()
+
+    /**
+     * Adapts this navigator to the media3 [Player] interface.
+     */
+    fun asPlayer(): Player
+}

readium/navigator/src/main/java/org/readium/r2/navigator/media3/api/SynchronizedMediaNavigator.kt

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2022 Readium Foundation. All rights reserved.
+ * Use of this source code is governed by the BSD-style license
+ * available in the top-level LICENSE file of the project.
+ */
+
+package org.readium.r2.navigator.media3.api
+
+import kotlinx.coroutines.flow.StateFlow
+import org.readium.r2.shared.ExperimentalReadiumApi
+import org.readium.r2.shared.publication.Locator
+
+/**
+ * A [MediaNavigator] aware of the utterances that are being read aloud.
+ */
+@ExperimentalReadiumApi
+interface SynchronizedMediaNavigator<P : MediaNavigator.Position> :
+    MediaNavigator<P> {
+
+    interface Utterance<P : MediaNavigator.Position> {
+        val text: String
+
+        val position: P
+
+        val range: IntRange?
+
+        val utteranceLocator: Locator
+
+        val tokenLocator: Locator?
+    }
+
+    val utterance: StateFlow<Utterance<P>>
+}

readium/navigator/src/main/java/org/readium/r2/navigator/media3/audio/AudioEngine.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2022 Readium Foundation. All rights reserved.
+ * Use of this source code is governed by the BSD-style license
+ * available in the top-level LICENSE file of the project.
+ */
+
+package org.readium.r2.navigator.media3.audio
+
+import androidx.media3.common.Player
+import kotlin.time.Duration
+import kotlinx.coroutines.flow.StateFlow
+import org.readium.r2.navigator.media3.api.MediaNavigator
+import org.readium.r2.navigator.preferences.Configurable
+import org.readium.r2.shared.ExperimentalReadiumApi
+
+@ExperimentalReadiumApi
+interface AudioEngine<S : Configurable.Settings, P : Configurable.Preferences<P>, E : AudioEngine.Error> :
+    Configurable<S, P> {
+
+    interface Error
+
+    data class Playback<E : Error>(
+        val state: MediaNavigator.State,
+        val playWhenReady: Boolean,
+        val error: E?
+    )
+
+    data class Position(
+        val index: Int,
+        val duration: Duration
+    )
+
+    val playback: StateFlow<Playback<E>>
+
+    val position: StateFlow<Position>
+
+    fun play()
+
+    fun pause()
+
+    fun seek(index: Long, position: Duration)
+
+    fun close()
+
+    fun asPlayer(): Player
+}