[FreshRSS] Fetch all articles from feeds, folders

jocmp · Jan 7, 2025 · dd58bdf · dd58bdf
1 parent 905ddb0
commit dd58bdf
Show file tree

Hide file tree

Showing 14 changed files with 269 additions and 93 deletions.
diff --git a/app/src/main/java/com/capyreader/app/ui/articles/ArticleLayout.kt b/app/src/main/java/com/capyreader/app/ui/articles/ArticleLayout.kt
@@ -299,7 +299,7 @@ fun ArticleLayout(
                 onFeedAdded = { onFeedAdded(it) },
                 onNavigateToSettings = onNavigateToSettings,
                 onFilterSelect = {
-                    if (!filter.hasArticlesSelected()) {
+                    if (!filter.hasArticleSelected()) {
                         openNextList { onSelectArticleFilter() }
                     } else {
                         closeDrawer()

diff --git a/app/src/main/java/com/capyreader/app/ui/articles/ArticleScreenViewModel.kt b/app/src/main/java/com/capyreader/app/ui/articles/ArticleScreenViewModel.kt
@@ -223,7 +223,7 @@ class ArticleScreenViewModel(
         refreshJob?.cancel()
 
         refreshJob = viewModelScope.launch(Dispatchers.IO) {
-            account.refresh().onFailure { throwable ->
+            account.refresh(latestFilter).onFailure { throwable ->
                 if (throwable is UnauthorizedError && _showUnauthorizedMessage == UnauthorizedMessageState.HIDE) {
                     _showUnauthorizedMessage = UnauthorizedMessageState.SHOW
                 }

diff --git a/app/src/main/java/com/capyreader/app/ui/articles/FeedList.kt b/app/src/main/java/com/capyreader/app/ui/articles/FeedList.kt
@@ -102,7 +102,7 @@ fun FeedList(
                     )
                 },
                 badge = { CountBadge(count = statusCount) },
-                selected = filter.hasArticlesSelected(),
+                selected = filter.hasArticleSelected(),
                 onClick = {
                     onFilterSelect()
                 }

diff --git a/capy/src/main/java/com/jocmp/capy/ArticleFilter.kt b/capy/src/main/java/com/jocmp/capy/ArticleFilter.kt
@@ -12,7 +12,7 @@ sealed class ArticleFilter(open val status: ArticleStatus) {
         return this is Feeds && this.feedID == feed.id && this.folderTitle.orEmpty() == feed.folderName
     }
 
-    fun hasArticlesSelected(): Boolean {
+    fun hasArticleSelected(): Boolean {
         return this is Articles
     }
 

diff --git a/capy/src/main/java/com/jocmp/capy/accounts/feedbin/FeedbinAccountDelegate.kt b/capy/src/main/java/com/jocmp/capy/accounts/feedbin/FeedbinAccountDelegate.kt
@@ -44,7 +44,7 @@ internal class FeedbinAccountDelegate(
 
     override suspend fun refresh(filter: ArticleFilter, cutoffDate: ZonedDateTime?): Result<Unit> {
         return try {
-            val since = articleRecords.maxUpdatedAt().toString()
+            val since = articleRecords.maxArrivedAt().toString()
 
             refreshFeeds()
             refreshTaggings()
@@ -190,7 +190,7 @@ internal class FeedbinAccountDelegate(
         Unit
     }
 
-    private suspend fun refreshArticles(since: String = articleRecords.maxUpdatedAt().toString()) {
+    private suspend fun refreshArticles(since: String = articleRecords.maxArrivedAt().toString()) {
         refreshStarredEntries()
         refreshUnreadEntries()
         refreshAllArticles(since = since)

diff --git a/capy/src/main/java/com/jocmp/capy/accounts/reader/ReaderAccountDelegate.kt b/capy/src/main/java/com/jocmp/capy/accounts/reader/ReaderAccountDelegate.kt
@@ -2,6 +2,7 @@ package com.jocmp.capy.accounts.reader
 
 import com.jocmp.capy.AccountDelegate
 import com.jocmp.capy.ArticleFilter
+import com.jocmp.capy.ArticleStatus
 import com.jocmp.capy.Feed
 import com.jocmp.capy.accounts.AddFeedResult
 import com.jocmp.capy.accounts.Source
@@ -26,6 +27,7 @@ import com.jocmp.readerclient.Stream
 import com.jocmp.readerclient.Subscription
 import com.jocmp.readerclient.SubscriptionEditAction
 import com.jocmp.readerclient.SubscriptionQuickAddResult
+import com.jocmp.readerclient.buildHexID
 import com.jocmp.readerclient.ext.editSubscription
 import com.jocmp.readerclient.ext.streamItemsIDs
 import kotlinx.coroutines.coroutineScope
@@ -48,31 +50,30 @@ internal class ReaderAccountDelegate(
 
     override suspend fun refresh(filter: ArticleFilter, cutoffDate: ZonedDateTime?): Result<Unit> {
         return withErrorHandling {
-            val since = articleRecords.maxUpdatedAt().toEpochSecond()
-
-            refreshFeeds()
-            refreshArticles(since = since)
+            refreshTopLevelArticles(filter)
+            val stream = filter.toStream(source)
+            refreshArticles(stream = stream, since = since(stream))
         }
     }
 
     override suspend fun markRead(articleIDs: List<String>): Result<Unit> {
         val results = articleIDs.chunked(MAX_CREATE_UNREAD_LIMIT).map { batchIDs ->
-            editTag(ids = batchIDs, addTag = Stream.READ)
+            editTag(ids = batchIDs, addTag = Stream.Read())
         }
 
         return results.firstOrNull { it.isFailure } ?: Result.success(Unit)
     }
 
     override suspend fun markUnread(articleIDs: List<String>): Result<Unit> {
-        return editTag(ids = articleIDs, removeTag = Stream.READ)
+        return editTag(ids = articleIDs, removeTag = Stream.Read())
     }
 
     override suspend fun addStar(articleIDs: List<String>): Result<Unit> {
-        return editTag(ids = articleIDs, addTag = Stream.STARRED)
+        return editTag(ids = articleIDs, addTag = Stream.Starred())
     }
 
     override suspend fun removeStar(articleIDs: List<String>): Result<Unit> {
-        return editTag(ids = articleIDs, removeTag = Stream.STARRED)
+        return editTag(ids = articleIDs, removeTag = Stream.Starred())
     }
 
     override suspend fun addFeed(
@@ -98,7 +99,7 @@ internal class ReaderAccountDelegate(
 
             if (feed != null) {
                 coroutineScope {
-                    launch { refreshArticles() }
+                    launch { refreshArticles(stream = Stream.ReadingList()) }
                 }
 
                 AddFeedResult.Success(feed)
@@ -199,6 +200,22 @@ internal class ReaderAccountDelegate(
         }
     }
 
+    private suspend fun refreshTopLevelArticles(filter: ArticleFilter) {
+        if (!filter.hasArticleSelected()) {
+            return
+        }
+
+        refreshFeeds()
+
+        if (filter.status != ArticleStatus.UNREAD) {
+            refreshStarredItems()
+        }
+
+        if (filter.status != ArticleStatus.STARRED) {
+            refreshUnreadItems()
+        }
+    }
+
     private fun upsertTaggings(subscription: Subscription) {
         subscription.categories.forEach { category ->
             database.taggingsQueries.upsert(
@@ -238,27 +255,29 @@ internal class ReaderAccountDelegate(
     }
 
     private suspend fun refreshArticles(
-        since: Long = articleRecords.maxUpdatedAt().toEpochSecond()
+        stream: Stream,
+        since: Long? = null
     ) {
-        refreshStarredItems()
-        refreshUnreadItems()
-        refreshAllArticles(since = since)
+        fetchPaginatedArticles(
+            since = since,
+            stream = stream
+        )
         fetchMissingArticles()
     }
 
     private suspend fun refreshUnreadItems() {
         withResult(
             googleReader.streamItemsIDs(
-                stream = Stream.READING_LIST,
-                excludedStream = Stream.READ
+                stream = Stream.ReadingList(),
+                excludedStream = Stream.Read()
             )
         ) { result ->
             articleRecords.markAllUnread(articleIDs = result.itemRefs.map { it.hexID })
         }
     }
 
     private suspend fun refreshStarredItems() {
-        withResult(googleReader.streamItemsIDs(stream = Stream.STARRED)) { result ->
+        withResult(googleReader.streamItemsIDs(stream = Stream.Starred())) { result ->
             articleRecords.markAllStarred(articleIDs = result.itemRefs.map { it.hexID })
         }
     }
@@ -272,7 +291,7 @@ internal class ReaderAccountDelegate(
                     val response = withPostToken {
                         googleReader.streamItemsContents(
                             postToken = postToken.get(),
-                            ids = chunkedIDs
+                            ids = chunkedIDs.map { buildHexID(it) }
                         )
                     }
 
@@ -284,14 +303,7 @@ internal class ReaderAccountDelegate(
         }
     }
 
-    private suspend fun refreshAllArticles(since: Long) {
-        fetchPaginatedItems(
-            since = since,
-            stream = Stream.READING_LIST
-        )
-    }
-
-    private suspend fun fetchPaginatedItems(
+    private suspend fun fetchPaginatedArticles(
         since: Long? = null,
         stream: Stream,
         continuation: String? = null,
@@ -317,7 +329,7 @@ internal class ReaderAccountDelegate(
 
         val nextContinuation = result.continuation ?: return
 
-        fetchPaginatedItems(
+        fetchPaginatedArticles(
             since = since,
             stream = stream,
             continuation = nextContinuation
@@ -358,7 +370,7 @@ internal class ReaderAccountDelegate(
                 articleRecords.updateStatus(
                     articleID = item.hexID,
                     updatedAt = updated,
-                    read = item.read
+                    read = item.read,
                 )
             }
         }
@@ -410,7 +422,6 @@ internal class ReaderAccountDelegate(
             postToken.set(googleReader.token().body())
         } catch (exception: IOException) {
             CapyLog.error(tag("post_token"), exception)
-            // continue
         }
     }
 
@@ -422,6 +433,14 @@ internal class ReaderAccountDelegate(
         return "${subscription.id}:${category.id}"
     }
 
+    private fun since(stream: Stream): Long? {
+        return if (stream.isStateStream) {
+            articleRecords.maxArrivedAt().toEpochSecond()
+        } else {
+            null
+        }
+    }
+
     companion object {
         const val MAX_PAGINATED_ITEM_LIMIT = 100
 
@@ -446,6 +465,25 @@ private val SubscriptionQuickAddResult.toSubscription: Subscription?
         )
     }
 
+private fun ArticleFilter.toStream(source: Source): Stream {
+    return when (this) {
+        is ArticleFilter.Articles -> Stream.Read()
+        is ArticleFilter.Feeds -> Stream.Feed(feedID)
+        is ArticleFilter.Folders -> folderStream(this, source)
+    }
+}
+
+/**
+ * Default to reading list for folders since Miniflux doesn't support label lookups
+ */
+private fun folderStream(filter: ArticleFilter.Folders, source: Source): Stream {
+    return if (source == Source.FRESHRSS) {
+        Stream.Label(filter.folderTitle)
+    } else {
+        Stream.ReadingList()
+    }
+}
+
 private fun userLabel(title: String): String {
     return "user/-/label/${title}"
 }

diff --git a/capy/src/main/java/com/jocmp/capy/persistence/ArticleRecords.kt b/capy/src/main/java/com/jocmp/capy/persistence/ArticleRecords.kt
@@ -178,7 +178,7 @@ internal class ArticleRecords internal constructor(
     }
 
     /** Date in UTC */
-    fun maxUpdatedAt(): ZonedDateTime {
+    fun maxArrivedAt(): ZonedDateTime {
         val max = database.articlesQueries.lastUpdatedAt().executeAsOne().MAX
 
         max ?: return cutoffDate()

diff --git a/capy/src/test/java/com/jocmp/capy/AccountTest.kt b/capy/src/test/java/com/jocmp/capy/AccountTest.kt
@@ -28,7 +28,7 @@ class AccountTest {
     @Before
     fun setup() {
         account = AccountFixture.create(parentFolder = folder)
-        coEvery { account.delegate.refresh(any()) }.returns(Result.success(Unit))
+        coEvery { account.delegate.refresh(any(), any()) }.returns(Result.success(Unit))
     }
 
     @Test

diff --git a/capy/src/test/java/com/jocmp/capy/OPMLFileTest.kt b/capy/src/test/java/com/jocmp/capy/OPMLFileTest.kt
@@ -5,7 +5,7 @@ import com.jocmp.capy.accounts.LocalAccountDelegate
 import com.jocmp.capy.db.Database
 import com.jocmp.capy.fixtures.AccountFixture
 import com.jocmp.capy.fixtures.FeedFixture
-import com.jocmp.capy.fixtures.TagFixture
+import com.jocmp.capy.fixtures.FolderFixture
 import io.mockk.mockk
 import kotlinx.coroutines.test.runTest
 import okhttp3.OkHttpClient
@@ -25,13 +25,13 @@ class OPMLFileTest {
     private lateinit var account: Account
     private lateinit var database: Database
     private lateinit var feedFixture: FeedFixture
-    private lateinit var tagFixture: TagFixture
+    private lateinit var tagFixture: FolderFixture
 
     @Before
     fun setup() {
         database = InMemoryDatabaseProvider.build(accountID)
         feedFixture = FeedFixture(database)
-        tagFixture = TagFixture(database)
+        tagFixture = FolderFixture(database)
 
         val delegate = LocalAccountDelegate(
             database = database,