Fetch missing articles

jocmp · Nov 7, 2024 · 5b41b7b · 5b41b7b
1 parent 02415ae
commit 5b41b7b
Show file tree

Hide file tree

Showing 17 changed files with 449 additions and 107 deletions.
diff --git a/capy/src/main/java/com/jocmp/capy/Account.kt b/capy/src/main/java/com/jocmp/capy/Account.kt
@@ -6,10 +6,9 @@ import com.jocmp.capy.accounts.LocalAccountDelegate
 import com.jocmp.capy.accounts.LocalOkHttpClient
 import com.jocmp.capy.accounts.Source
 import com.jocmp.capy.accounts.asOPML
+import com.jocmp.capy.accounts.reader.buildFreshRSSDelegate
 import com.jocmp.capy.accounts.feedbin.FeedbinAccountDelegate
 import com.jocmp.capy.accounts.feedbin.FeedbinOkHttpClient
-import com.jocmp.capy.accounts.reader.ReaderAccountDelegate
-import com.jocmp.capy.accounts.reader.ReaderOkHttpClient
 import com.jocmp.capy.articles.UnreadSortOrder
 import com.jocmp.capy.common.TimeHelpers.nowUTC
 import com.jocmp.capy.common.sortedByTitle
@@ -20,7 +19,6 @@ import com.jocmp.capy.opml.OPMLImporter
 import com.jocmp.capy.persistence.ArticleRecords
 import com.jocmp.capy.persistence.FeedRecords
 import com.jocmp.feedbinclient.Feedbin
-import com.jocmp.readerclient.GoogleReader
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.flow.map
@@ -258,23 +256,6 @@ data class Account(
 private fun Feedbin.Companion.forAccount(path: URI, preferences: AccountPreferences) =
     create(client = FeedbinOkHttpClient.forAccount(path, preferences))
 
-private fun buildFreshRSSDelegate(
-    database: Database,
-    path: URI,
-    preferences: AccountPreferences
-): AccountDelegate {
-    val httpClient = ReaderOkHttpClient.forAccount(path, preferences)
-
-    return ReaderAccountDelegate(
-        database = database,
-        httpClient = httpClient,
-        googleReader = GoogleReader.create(
-            client = httpClient,
-            baseURL = preferences.url.get()
-        )
-    )
-}
-
 private fun AutoDelete.cutoffDate(): ZonedDateTime? {
     val now = nowUTC()
 

diff --git a/...cmp/feedbinclient/BasicAuthInterceptor.kt → ...cmp/capy/accounts/BasicAuthInterceptor.kt b/...cmp/feedbinclient/BasicAuthInterceptor.kt → ...cmp/capy/accounts/BasicAuthInterceptor.kt
@@ -1,12 +1,12 @@
-package com.jocmp.feedbinclient
+package com.jocmp.capy.accounts
 
 import okhttp3.Interceptor
 
 class BasicAuthInterceptor(private val credentials: () -> String) : Interceptor {
     override fun intercept(chain: Interceptor.Chain): okhttp3.Response {
         val request = chain.request()
 
-        if (request.headers("Authorization").isNullOrEmpty()) {
+        if (request.headers("Authorization").isEmpty()) {
             val authenticatedRequest =
                 request.newBuilder().header("Authorization", credentials()).build()
             return chain.proceed(authenticatedRequest)

diff --git a/capy/src/main/java/com/jocmp/capy/accounts/feedbin/FeedbinAccountDelegate.kt b/capy/src/main/java/com/jocmp/capy/accounts/feedbin/FeedbinAccountDelegate.kt
@@ -283,7 +283,7 @@ internal class FeedbinAccountDelegate(
         coroutineScope {
             ids.chunked(MAX_ENTRY_LIMIT).map { chunkedIDs ->
                 launch {
-                    fetchPaginatedEntries(ids = chunkedIDs)
+                    fetchPaginatedEntries(ids = chunkedIDs.map { it.toLong() })
                 }
             }
         }

diff --git a/capy/src/main/java/com/jocmp/capy/accounts/feedbin/FeedbinOkHttpClient.kt b/capy/src/main/java/com/jocmp/capy/accounts/feedbin/FeedbinOkHttpClient.kt
@@ -2,7 +2,7 @@ package com.jocmp.capy.accounts.feedbin
 
 import com.jocmp.capy.AccountPreferences
 import com.jocmp.capy.accounts.httpClientBuilder
-import com.jocmp.feedbinclient.BasicAuthInterceptor
+import com.jocmp.capy.accounts.BasicAuthInterceptor
 import okhttp3.Credentials
 import okhttp3.OkHttpClient
 import java.net.URI

diff --git a/capy/src/main/java/com/jocmp/capy/accounts/reader/BuildFreshRSSDelegate.kt b/capy/src/main/java/com/jocmp/capy/accounts/reader/BuildFreshRSSDelegate.kt
@@ -0,0 +1,25 @@
+package com.jocmp.capy.accounts.reader
+
+import com.jocmp.capy.AccountDelegate
+import com.jocmp.capy.AccountPreferences
+import com.jocmp.capy.accounts.httpClientBuilder
+import com.jocmp.capy.db.Database
+import com.jocmp.readerclient.GoogleReader
+import java.net.URI
+
+internal fun buildFreshRSSDelegate(
+    database: Database,
+    path: URI,
+    preferences: AccountPreferences
+): AccountDelegate {
+    val httpClient = ReaderOkHttpClient.forAccount(path, preferences)
+
+    return ReaderAccountDelegate(
+        database = database,
+        httpClient = httpClientBuilder(cachePath = path).build(),
+        googleReader = GoogleReader.create(
+            client = httpClient,
+            baseURL = preferences.url.get()
+        )
+    )
+}
diff --git a/capy/src/main/java/com/jocmp/capy/accounts/reader/ReaderAccountDelegate.kt b/capy/src/main/java/com/jocmp/capy/accounts/reader/ReaderAccountDelegate.kt
@@ -13,23 +13,26 @@ import com.jocmp.capy.db.Database
 import com.jocmp.capy.persistence.ArticleRecords
 import com.jocmp.readerclient.Category
 import com.jocmp.readerclient.GoogleReader
+import com.jocmp.readerclient.GoogleReader.Companion.BAD_TOKEN_HEADER_KEY
 import com.jocmp.readerclient.Item
+import com.jocmp.readerclient.ItemRef
 import com.jocmp.readerclient.Stream
 import com.jocmp.readerclient.Subscription
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.launch
 import okhttp3.OkHttpClient
 import org.jsoup.Jsoup
+import retrofit2.Response
 import java.io.IOException
 import java.time.ZonedDateTime
+import java.util.concurrent.atomic.AtomicReference
 
-/**
- * Save Auth Token for later use
- * self.credentials = Credentials(type: .readerAPIKey, username: credentials.username, secret: authString)
- */
 internal class ReaderAccountDelegate(
     private val database: Database,
-    private val httpClient: OkHttpClient,
     private val googleReader: GoogleReader,
+    httpClient: OkHttpClient = OkHttpClient(),
 ) : AccountDelegate {
+    private var postToken = AtomicReference<String?>(null)
     private val articleContent = ArticleContent(httpClient)
     private val articleRecords = ArticleRecords(database)
 
@@ -120,7 +123,7 @@ internal class ReaderAccountDelegate(
             title = subscription.title,
             feed_url = subscription.url,
             site_url = subscription.htmlUrl,
-            favicon_url = subscription.iconUrl
+            favicon_url = subscription.iconUrl.ifBlank { null }
         )
     }
 
@@ -139,8 +142,8 @@ internal class ReaderAccountDelegate(
     private suspend fun refreshArticles(
         since: Long = articleRecords.maxUpdatedAt().toEpochSecond()
     ) {
-        refreshUnreadItems()
         refreshStarredItems()
+        refreshUnreadItems()
         refreshAllArticles(since = since)
         fetchMissingArticles()
     }
@@ -162,8 +165,25 @@ internal class ReaderAccountDelegate(
         }
     }
 
+    private suspend fun fetchMissingArticles() {
+        val ids = articleRecords.findMissingArticles()
+
+        coroutineScope {
+            ids.chunked(MAX_PAGINATED_ITEM_LIMIT).map { chunkedIDs ->
+                launch {
+                    val response = withPostToken {
+                        googleReader.streamItemsContents(
+                            postToken = postToken.get(),
+                            ids = chunkedIDs
+                        )
+                    }
+
+                    val result = response.body() ?: return@launch
 
-    private fun fetchMissingArticles() {
+                    saveItems(result.items)
+                }
+            }
+        }
     }
 
     private suspend fun refreshAllArticles(since: Long) {
@@ -178,15 +198,21 @@ internal class ReaderAccountDelegate(
         stream: Stream,
         continuation: String? = null,
     ) {
-        val response = googleReader.streamContents(
+        val response = googleReader.streamItemsIDs(
             streamID = stream.id,
             since = since,
             continuation = continuation,
+            excludedStreamID = Stream.READ.id,
+            count = MAX_PAGINATED_ITEM_LIMIT,
         )
 
         val result = response.body() ?: return
 
-        saveItems(result.items)
+        coroutineScope {
+            launch {
+                fetchItemContents(result.itemRefs)
+            }
+        }
 
         val nextContinuation = result.continuation ?: return
 
@@ -197,6 +223,19 @@ internal class ReaderAccountDelegate(
         )
     }
 
+    private suspend fun fetchItemContents(items: List<ItemRef>) {
+        val response = withPostToken {
+            googleReader.streamItemsContents(
+                postToken = postToken.get(),
+                ids = items.map { it.hexID }
+            )
+        }
+
+        val result = response.body() ?: return
+
+        saveItems(result.items)
+    }
+
     private fun saveItems(items: List<Item>) {
         database.transactionWithErrorHandling {
             items.forEach { item ->
@@ -207,7 +246,7 @@ internal class ReaderAccountDelegate(
                     feed_id = item.origin.streamId,
                     title = item.title,
                     author = item.author,
-                    content_html = item.summary.content,
+                    content_html = item.content?.content ?: item.summary.content,
                     extracted_content_url = null,
                     summary = Jsoup.parse(item.summary.content).text(),
                     url = item.canonical.firstOrNull()?.href,
@@ -224,7 +263,34 @@ internal class ReaderAccountDelegate(
         }
     }
 
+    private suspend fun <T> withPostToken(handler: suspend () -> Response<T>): Response<T> {
+        val response = handler()
+
+        val isBadToken = response
+            .headers()
+            .get(BAD_TOKEN_HEADER_KEY)
+            .orEmpty()
+            .toBoolean()
+
+        if (!isBadToken) {
+            return response
+        }
+
+        try {
+            postToken.set(googleReader.token().body())
+
+            return handler()
+        } catch (exception: IOException) {
+            return response
+        }
+    }
+
     private fun taggingID(subscription: Subscription, category: Category): String {
         return "${subscription.id}:${category.id}"
     }
+
+
+    companion object {
+        const val MAX_PAGINATED_ITEM_LIMIT = 100
+    }
 }
diff --git a/capy/src/main/java/com/jocmp/capy/accounts/reader/ReaderOkHttpClient.kt b/capy/src/main/java/com/jocmp/capy/accounts/reader/ReaderOkHttpClient.kt
@@ -1,8 +1,8 @@
 package com.jocmp.capy.accounts.reader
 
 import com.jocmp.capy.AccountPreferences
+import com.jocmp.capy.accounts.BasicAuthInterceptor
 import com.jocmp.capy.accounts.httpClientBuilder
-import com.jocmp.feedbinclient.BasicAuthInterceptor
 import okhttp3.OkHttpClient
 import java.net.URI
 

diff --git a/capy/src/main/java/com/jocmp/capy/persistence/ArticleRecords.kt b/capy/src/main/java/com/jocmp/capy/persistence/ArticleRecords.kt
@@ -33,12 +33,11 @@ internal class ArticleRecords internal constructor(
         ).executeAsOneOrNull()
     }
 
-    fun findMissingArticles(): List<Long> {
+    fun findMissingArticles(): List<String> {
         return database
             .articlesQueries
             .findMissingArticles()
             .executeAsList()
-            .map { it.toLong() }
     }
 
     internal suspend fun notifications(since: ZonedDateTime): List<ArticleNotification> {