Sync element comments

app/src/androidMain/kotlin/api/Api.kt

@@ -4,6 +4,8 @@ import area.AreaJson
 import area.toAreasJson
 import element.ElementJson
 import element.toElementsJson
+import element_comment.ElementCommentJson
+import element_comment.toElementCommentsJson
 import event.EventJson
 import event.toEventsJson
 import kotlinx.coroutines.Dispatchers
@@ -26,6 +28,11 @@ private val BASE_URL = "https://api.btcmap.org".toHttpUrl()
 interface Api {
     suspend fun getAreas(updatedSince: ZonedDateTime?, limit: Long): List<AreaJson>
 
+    suspend fun getElementComments(
+        updatedSince: ZonedDateTime?,
+        limit: Long
+    ): List<ElementCommentJson>
+
     suspend fun getElements(updatedSince: ZonedDateTime?, limit: Long): List<ElementJson>
 
     suspend fun getEvents(updatedSince: ZonedDateTime?, limit: Long): List<EventJson>
@@ -58,6 +65,28 @@ class ApiImpl(
         }
     }
 
+    override suspend fun getElementComments(
+        updatedSince: ZonedDateTime?,
+        limit: Long
+    ): List<ElementCommentJson> {
+        val url = baseUrl.newBuilder().apply {
+            addPathSegment("v3")
+            addPathSegment("element-comments")
+            addQueryParameter("updated_since", updatedSince.apiFormat())
+            addQueryParameter("limit", "$limit")
+        }.build()
+
+        val res = httpClient.newCall(Request.Builder().url(url).build()).executeAsync()
+
+        if (!res.isSuccessful) {
+            throw Exception("Unexpected HTTP response code: ${res.code}")
+        }
+
+        return withContext(Dispatchers.IO) {
+            res.body.byteStream().use { it.toElementCommentsJson() }
+        }
+    }
+
     override suspend fun getElements(updatedSince: ZonedDateTime?, limit: Long): List<ElementJson> {
         val url = baseUrl.newBuilder().apply {
             addPathSegment("v3")

app/src/androidMain/kotlin/app/AppModule.kt

@@ -35,9 +35,11 @@ import sync.SyncNotificationController
 import user.UserQueries
 import user.UsersModel
 import user.UsersRepo
+import element_comment.ElementCommentQueries
+import element_comment.ElementCommentRepo
 
 val appModule = module {
-    single { Database(get<Context>().getDatabasePath("btcmap-2024-05-15.db").absolutePath).conn }
+    single { Database(get<Context>().getDatabasePath("btcmap-2024-10-18.db").absolutePath).conn }
 
     single { ApiImpl() }.bind(Api::class)
 
@@ -55,6 +57,9 @@ val appModule = module {
     singleOf(::ElementQueries)
     singleOf(::ElementsRepo)
 
+    singleOf(::ElementCommentQueries)
+    singleOf(::ElementCommentRepo)
+
     singleOf(::ConfQueries)
     singleOf(::ConfRepo)
 

app/src/androidMain/kotlin/db/Database.kt

@@ -10,6 +10,7 @@ import androidx.sqlite.use
 import area.AreaQueries
 import conf.ConfQueries
 import element.ElementQueries
+import element_comment.ElementCommentQueries
 import event.EventQueries
 import kotlinx.coroutines.flow.MutableStateFlow
 import reports.ReportQueries
@@ -35,6 +36,7 @@ class Database(path: String) {
                 }
 
                 if (version == 0) {
+                    execSQL(ElementCommentQueries.CREATE_TABLE)
                     execSQL(ElementQueries.CREATE_TABLE)
                     execSQL(EventQueries.CREATE_TABLE)
                     execSQL(ReportQueries.CREATE_TABLE)

app/src/androidMain/kotlin/element/ElementsRepo.kt

@@ -181,6 +181,7 @@ class ElementsRepo(
                     .map { it.toElement() })
             }
         }
+        elementsUpdatedAt.update { LocalDateTime.now() }
     }
 
     suspend fun sync(): SyncReport {

app/src/androidMain/kotlin/element_comment/ElementComment.kt

@@ -0,0 +1,9 @@
+package element_comment
+
+data class ElementComment(
+    val id: Long,
+    val elementId: Long,
+    val comment: String,
+    val createdAt: String,
+    val updatedAt: String,
+)

app/src/androidMain/kotlin/element_comment/ElementCommentJson.kt

@@ -0,0 +1,36 @@
+package element_comment
+
+import json.toJsonArray
+import java.io.InputStream
+
+data class ElementCommentJson(
+    val id: Long,
+    val elementId: Long?,
+    val comment: String?,
+    val createdAt: String?,
+    val updatedAt: String,
+    val deletedAt: String?,
+)
+
+fun ElementCommentJson.toElementComment(): ElementComment {
+    return ElementComment(
+        id = id,
+        elementId = elementId!!,
+        comment = comment!!,
+        createdAt = createdAt!!,
+        updatedAt = updatedAt,
+    )
+}
+
+fun InputStream.toElementCommentsJson(): List<ElementCommentJson> {
+    return toJsonArray().map {
+        ElementCommentJson(
+            id = it.getLong("id"),
+            elementId = it.optLong("element_id"),
+            comment = it.optString("comment").ifBlank { null },
+            createdAt = it.optString("created_at").ifBlank { null },
+            updatedAt = it.getString("created_at"),
+            deletedAt = it.optString("deleted_at").ifBlank { null },
+        )
+    }
+}

app/src/androidMain/kotlin/element_comment/ElementCommentQueries.kt

@@ -0,0 +1,132 @@
+package element_comment
+
+import androidx.sqlite.SQLiteConnection
+import androidx.sqlite.use
+import db.getZonedDateTime
+import db.transaction
+import java.time.ZonedDateTime
+
+class ElementCommentQueries(private val conn: SQLiteConnection) {
+
+    companion object {
+        const val CREATE_TABLE = """
+            CREATE TABLE element_comment (
+                id INTEGER NOT NULL PRIMARY KEY,
+                element_id INTEGER NOT NULL,
+                comment TEXT NOT NULL,
+                created_at TEXT NOT NULL,
+                updated_at TEXT NOT NULL
+            );
+            """
+    }
+
+    fun insertOrReplace(comments: List<ElementComment>) {
+        conn.transaction { conn ->
+            comments.forEach { comment ->
+                conn.prepare(
+                    """
+                    INSERT OR REPLACE
+                    INTO element_comment (
+                        id,
+                        element_id,
+                        comment,
+                        created_at,
+                        updated_at
+                    ) VALUES (?1, ?2, ?3, ?4, ?5)
+                    """
+                ).use {
+                    it.bindLong(1, comment.id)
+                    it.bindLong(2, comment.elementId)
+                    it.bindText(3, comment.comment)
+                    it.bindText(4, comment.createdAt)
+                    it.bindText(5, comment.updatedAt)
+                    it.step()
+                }
+            }
+        }
+    }
+
+    fun selectById(id: Long): ElementComment? {
+        return conn.prepare(
+            """
+                SELECT
+                    id,
+                    element_id,
+                    comment,
+                    created_at,
+                    updated_at
+                FROM element_comment
+                WHERE id = ?1
+                """
+        ).use {
+            it.bindLong(1, id)
+
+            if (it.step()) {
+                ElementComment(
+                    id = it.getLong(0),
+                    elementId = it.getLong(1),
+                    comment = it.getText(2),
+                    createdAt = it.getText(3),
+                    updatedAt = it.getText(4),
+                )
+            } else {
+                null
+            }
+        }
+    }
+
+    fun selectByElementId(elementId: Long): List<ElementComment> {
+        return conn.prepare(
+            """
+                SELECT
+                    id,
+                    element_id,
+                    comment,
+                    created_at,
+                    updated_at
+                FROM element_comment
+                WHERE element_id = ?1
+                """
+        ).use {
+            it.bindLong(1, elementId)
+
+            buildList {
+                while (it.step()) {
+                    add(
+                        ElementComment(
+                            id = it.getLong(0),
+                            elementId = it.getLong(1),
+                            comment = it.getText(2),
+                            createdAt = it.getText(3),
+                            updatedAt = it.getText(4),
+                        )
+                    )
+                }
+            }
+        }
+    }
+
+    fun selectMaxUpdatedAt(): ZonedDateTime? {
+        return conn.prepare("SELECT max(updated_at) FROM element_comment").use {
+            if (it.step()) {
+                it.getZonedDateTime(0)
+            } else {
+                null
+            }
+        }
+    }
+
+    fun selectCount(): Long {
+        return conn.prepare("SELECT count(*) FROM element_comment").use {
+            it.step()
+            it.getLong(0)
+        }
+    }
+
+    fun deleteById(id: Long) {
+        conn.prepare("DELETE FROM element_comment WHERE id = ?1").use {
+            it.bindLong(1, id)
+            it.step()
+        }
+    }
+}

app/src/androidMain/kotlin/element_comment/ElementCommentRepo.kt

@@ -0,0 +1,108 @@
+package element_comment
+
+import android.app.Application
+import api.Api
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import java.time.Duration
+import java.time.ZoneOffset
+import java.time.ZonedDateTime
+
+class ElementCommentRepo(
+    private val api: Api,
+    private val app: Application,
+    private val queries: ElementCommentQueries,
+) {
+
+    suspend fun selectByElementId(elementId: Long): List<ElementComment> {
+        return withContext(Dispatchers.IO) {
+            queries.selectByElementId(elementId)
+        }
+    }
+
+    suspend fun selectCount(): Long {
+        return withContext(Dispatchers.IO) {
+            queries.selectCount()
+        }
+    }
+
+    suspend fun hasBundledElements(): Boolean {
+        return withContext(Dispatchers.IO) {
+            app.resources.assets.list("")!!.contains("element-comments.json")
+        }
+    }
+
+    suspend fun fetchBundledElements() {
+        withContext(Dispatchers.IO) {
+            app.assets.open("element-comments.json").use { bundledElements ->
+                queries.insertOrReplace(bundledElements
+                    .toElementCommentsJson()
+                    .filter { it.deletedAt == null }
+                    .map { it.toElementComment() })
+            }
+        }
+    }
+
+    suspend fun sync(): SyncReport {
+        return withContext(Dispatchers.IO) {
+            val startedAt = ZonedDateTime.now(ZoneOffset.UTC)
+            var newItems = 0L
+            var updatedItems = 0L
+            var deletedItems = 0L
+            var maxKnownUpdatedAt = queries.selectMaxUpdatedAt()
+
+            while (true) {
+                val delta = api.getElementComments(maxKnownUpdatedAt, BATCH_SIZE)
+
+                if (delta.isEmpty()) {
+                    break
+                } else {
+                    maxKnownUpdatedAt = ZonedDateTime.parse(delta.maxBy { it.updatedAt }.updatedAt)
+                }
+
+                delta.forEach {
+                    val cached = queries.selectById(it.id)
+
+                    if (it.deletedAt == null) {
+                        if (cached == null) {
+                            newItems++
+                        } else {
+                            updatedItems++
+                        }
+
+                        queries.insertOrReplace(listOf(it.toElementComment()))
+                    } else {
+                        if (cached == null) {
+                            // Already evicted from cache, nothing to do here
+                        } else {
+                            queries.deleteById(it.id)
+                            deletedItems++
+                        }
+                    }
+                }
+
+                if (delta.size < BATCH_SIZE) {
+                    break
+                }
+            }
+
+            SyncReport(
+                duration = Duration.between(startedAt, ZonedDateTime.now(ZoneOffset.UTC)),
+                newElementComments = newItems,
+                updatedElementComments = updatedItems,
+                deletedElementComments = deletedItems,
+            )
+        }
+    }
+
+    data class SyncReport(
+        val duration: Duration,
+        val newElementComments: Long,
+        val updatedElementComments: Long,
+        val deletedElementComments: Long,
+    )
+
+    companion object {
+        private const val BATCH_SIZE = 5000L
+    }
+}