GH-656 Simplify package document to improve querying possibilities

hub-backend/src/main/kotlin/org/panda_lang/hub/packages/InMemoryPackageRepository.kt

@@ -17,25 +17,31 @@
 package org.panda_lang.hub.packages
 
 import org.panda_lang.hub.github.RepositoryId
+import org.panda_lang.hub.shared.Date
+import org.panda_lang.hub.shared.paging.Page
 import org.panda_lang.hub.user.UserId
+import org.panda_lang.hub.utils.mapOnly
 import java.util.concurrent.ConcurrentHashMap
 
 internal class InMemoryPackageRepository : PackageRepository {
 
     private val packages = ConcurrentHashMap<PackageId, Package>()
 
+    override suspend fun findLatest(page: Int, pageSize: Int): Page<Package> {
+        TODO("Not yet implemented")
+    }
+
+    override suspend fun findPopular(page: Int, pageSize: Int): Page<Package> {
+        TODO("Not yet implemented")
+    }
+
     override suspend fun updateDailyStats(packageId: PackageId, date: Date, dailyBulk: Map<Country, Int>) =
-        packages[packageId]!!.let {
-            packages[packageId] = Package(
-                _id = it._id,
-                registered = it.registered,
-                repository = it.repository,
-                dailyStats = HashMap(it.dailyStats).also { updatedStats ->
-                    updatedStats[date.toString()] = ((updatedStats[date.toString()] ?: DailyStats()).countries.asSequence() + dailyBulk.asSequence())
-                        .groupBy({ entry -> entry.key }, { entry -> entry.value })
-                        .mapValues { entry -> entry.value.sum() }
-                        .let { map -> DailyStats(map) }
-                }
+        packages[packageId]!!.let { pkg ->
+            packages[packageId] = pkg.update(
+                dailyStats = pkg.dailyStats.mapOnly(
+                    { it.date == date },
+                    { daily -> DailyStats(date, daily.country, daily.requests + (dailyBulk[daily.country] ?: 0)) }
+                )
             )
         }
 

hub-backend/src/main/kotlin/org/panda_lang/hub/packages/MongoPackageRepository.kt

@@ -16,24 +16,39 @@
 
 package org.panda_lang.hub.packages
 
+import org.litote.kmongo.and
+import org.litote.kmongo.bson
 import org.litote.kmongo.coroutine.CoroutineCollection
 import org.litote.kmongo.div
 import org.litote.kmongo.eq
 import org.litote.kmongo.inc
-import org.litote.kmongo.keyProjection
 import org.litote.kmongo.upsert
 import org.panda_lang.hub.github.GitHubRepositoryInfo
 import org.panda_lang.hub.github.GitHubUserInfo
 import org.panda_lang.hub.github.RepositoryId
+import org.panda_lang.hub.shared.Date
+import org.panda_lang.hub.shared.paging.Page
 import org.panda_lang.hub.user.UserId
+import org.panda_lang.hub.utils.page
+import org.panda_lang.hub.utils.posOp
 
 internal class MongoPackageRepository(private val collection: CoroutineCollection<Package>) : PackageRepository {
 
+    override suspend fun findLatest(page: Int, pageSize: Int): Page<Package> =
+        collection.find()
+            .ascendingSort(Package::registeredAt)
+            .page(collection, page, pageSize)
+
+    override suspend fun findPopular(page: Int, pageSize: Int): Page<Package> =
+        collection.find()
+            .sort("{ \$sum: 'dailyStats.\$.countries.\$' }".bson)
+            .page(collection, page, pageSize)
+
     override suspend fun updateDailyStats(packageId: PackageId, date: Date, dailyBulk: Map<Country, Int>) =
         dailyBulk.forEach {
             collection.updateOneById(
-                packageId,
-                inc(Package::dailyStats.keyProjection(date.toString()) / DailyStats::countries.keyProjection(it.key), it.value),
+                and(Package::_id eq packageId, Package::dailyStats / DailyStats::date eq date, Package::dailyStats / DailyStats::country eq it.key),
+                inc(Package::dailyStats.posOp / DailyStats::requests, it.value),
                 upsert()
             )
         }

hub-backend/src/main/kotlin/org/panda_lang/hub/packages/Package.kt

@@ -18,49 +18,37 @@ package org.panda_lang.hub.packages
 
 import kotlinx.serialization.Serializable
 import org.panda_lang.hub.github.GitHubRepositoryInfo
-import java.util.Calendar
+import org.panda_lang.hub.shared.Date
 
 typealias PackageId = String
-typealias DateId = String
 typealias Country = String
 typealias RequestsCount = Int
 
 @Serializable
 data class Package(
     val _id: PackageId,
-    val registered: Boolean = false,
     val repository: GitHubRepositoryInfo,
-    val dailyStats: Map<DateId, DailyStats> = emptyMap()
+    val registeredAt: Date? = null,
+    val dailyStats: List<DailyStats> = emptyList()
 ) {
 
-    fun toRegistered() = Package(
-        _id = _id,
-        registered = true,
-        repository = repository
+    fun update(
+        id: PackageId = this._id,
+        repository: GitHubRepositoryInfo = this.repository,
+        registeredAt: Date? = this.registeredAt,
+        dailyStats: List<DailyStats> = this.dailyStats
+    ) = Package(
+        _id = id,
+        repository = repository,
+        registeredAt = registeredAt,
+        dailyStats = dailyStats
     )
 
 }
 
-@Serializable
-data class Date(
-    val year: Int,
-    val month: Int,
-    val day: Int
-) {
-
-    companion object {
-
-        fun now() = Calendar.getInstance().let {
-            Date(it.get(Calendar.YEAR), it.get(Calendar.MONTH), it.get(Calendar.DAY_OF_MONTH))
-        }
-
-    }
-
-    override fun toString(): DateId = "$year-$month-$day"
-
-}
-
 @Serializable
 data class DailyStats(
-    val countries: Map<Country, RequestsCount> = emptyMap()
+    val date: Date = Date.now(),
+    val country: Country,
+    val requests: RequestsCount = 0
 )

hub-backend/src/main/kotlin/org/panda_lang/hub/packages/PackageRepository.kt

@@ -17,10 +17,18 @@
 package org.panda_lang.hub.packages
 
 import org.panda_lang.hub.github.RepositoryId
+import org.panda_lang.hub.shared.Date
+import org.panda_lang.hub.shared.paging.Page
 import org.panda_lang.hub.user.UserId
 
 internal interface PackageRepository {
 
+    suspend fun findPopular(page: Int, pageSize: Int): Page<Package>
+
+    // suspend fun findTrending(page: Int, pageSize: Int): Page<Package>
+
+    suspend fun findLatest(page: Int, pageSize: Int): Page<Package>
+
     suspend fun updateDailyStats(packageId: PackageId, date: Date, dailyBulk: Map<Country, Int>)
 
     suspend fun savePackage(pkg: Package): Package

hub-backend/src/main/kotlin/org/panda_lang/hub/packages/PackageRouter.kt

@@ -32,10 +32,19 @@ import org.panda_lang.hub.utils.principal
 import org.panda_lang.hub.utils.respond
 import org.panda_lang.hub.utils.respondIf
 
-@Location("/repositories/{login}")
+@Location("/packages/trending/{page}")
+internal data class TrendingLocation(val page: Int)
+
+@Location("/packages/popular/{page}")
+internal data class PopularLocation(val page: Int)
+
+@Location("/packages/latest/{page}")
+internal data class LatestLocation(val page: Int)
+
+@Location("/packages/repositories/{login}")
 internal class RepositoriesLocation(val login: String)
 
-@Location("/packages/{login}")
+@Location("/packages/repositories/{login}")
 internal class PackagesLocation(val login: String)
 
 @Location("/package/{login}/{name}")
@@ -45,7 +54,7 @@ internal class PackageLocation(val login: String, val name: String)
 internal class VersionsLocation(val login: String, val name: String)
 
 @Location("/package/{login}/{name}/latest")
-internal class LatestLocation(val login: String, val name: String)
+internal class LatestLVersionocation(val login: String, val name: String)
 
 internal fun Routing.routes(packageFacade: PackageFacade) {
     get <RepositoriesLocation> { repositoriesLocation ->
@@ -67,7 +76,6 @@ internal fun Routing.routes(packageFacade: PackageFacade) {
             ErrorResponseException(HttpStatusCode.NotFound)
         }
     }
-
     authenticate("jwt") {
         post<PackageLocation> { packageLocation ->
             principal <JWTPrincipal> {

hub-backend/src/main/kotlin/org/panda_lang/hub/packages/PackageService.kt

@@ -18,6 +18,7 @@ package org.panda_lang.hub.packages
 
 import org.panda_lang.hub.github.GitHubClient
 import org.panda_lang.hub.github.RepositoryId
+import org.panda_lang.hub.shared.Date
 import org.panda_lang.hub.user.UserFacade
 
 internal class PackageService(
@@ -34,7 +35,11 @@ internal class PackageService(
 
     suspend fun getOrFetchPackage(id: RepositoryId): Package =
         getAnyPackage(id).let {
-            if (it.registered) it else packageRepository.savePackage(it.toRegistered())
+            if (it.registeredAt != null) it else packageRepository.savePackage(
+                it.update(
+                    registeredAt = Date.now()
+                )
+            )
         }
 
     suspend fun getAllPackages(login: String): List<Package> =

hub-backend/src/main/kotlin/org/panda_lang/hub/packages/StatsService.kt

@@ -18,6 +18,7 @@ package org.panda_lang.hub.packages
 
 import com.maxmind.geoip2.DatabaseReader
 import com.maxmind.geoip2.exception.AddressNotFoundException
+import org.panda_lang.hub.shared.Date
 import org.panda_lang.hub.utils.pollWhile
 import java.net.InetAddress
 import java.util.concurrent.ConcurrentLinkedQueue

hub-backend/src/main/kotlin/org/panda_lang/hub/shared/Date.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2021 Hub Team of panda-lang organization
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.panda_lang.hub.shared
+
+import kotlinx.serialization.Serializable
+import java.util.Calendar
+
+typealias DateId = String
+
+@Serializable
+data class Date(
+    val year: Int,
+    val month: Int,
+    val day: Int
+) {
+
+    companion object {
+
+        fun now() = Calendar.getInstance().let {
+            Date(it.get(Calendar.YEAR), it.get(Calendar.MONTH), it.get(Calendar.DAY_OF_MONTH))
+        }
+
+    }
+
+    override fun toString(): DateId = "$year-${month.asDateUnit()}-${day.asDateUnit()}"
+
+}
+
+private fun Int.asDateUnit() = this.toString().padStart(2, '0')

hub-backend/src/main/kotlin/org/panda_lang/hub/shared/paging/Page.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2021 Hub Team of panda-lang organization
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.panda_lang.hub.shared.paging
+
+class Page<T>(
+    val content: List<T>,
+    val pageSize: Int,
+    val pageNumber: Int,
+    val pagesCount: Int
+)

hub-backend/src/main/kotlin/org/panda_lang/hub/utils/CollectionExtensions.kt

@@ -31,3 +31,7 @@ fun <T> Queue<T>.pollWhile(predicate: (Queue<T>) -> Boolean): List<T> {
 
     return result
 }
+
+fun <T> Iterable<T>.mapOnly(predicate: (T) -> Boolean, function: (T) -> T): List<T> = map {
+    if (predicate.invoke(it)) function.invoke(it) else it
+}.toList()

hub-backend/src/main/kotlin/org/panda_lang/hub/utils/MongoExtensions.kt

@@ -16,6 +16,34 @@
 
 package org.panda_lang.hub.utils
 
-fun String.encodeMongo(): String = replace("\\", "\\\\").replace("\$", "\\u0024").replace(".", "\\u002e")
+import org.litote.kmongo.colProperty
+import org.litote.kmongo.coroutine.CoroutineCollection
+import org.litote.kmongo.coroutine.CoroutineFindPublisher
+import org.litote.kmongo.property.KPropertyPath
+import org.panda_lang.hub.shared.paging.Page
+import kotlin.math.ceil
+import kotlin.reflect.KProperty1
 
-fun String.decodeMongo(): String = replace("\\u002e", ".").replace("\\u0024", "\$").replace("\\\\", "\\")
+fun String.encodeMongo() = replace("\\", "\\\\")
+    .replace("\$", "\\u0024")
+    .replace(".", "\\u002e")
+
+fun String.decodeMongo() = replace("\\u002e", ".")
+    .replace("\\u0024", "\$")
+    .replace("\\\\", "\\")
+
+@Suppress("UNCHECKED_CAST")
+val <T> KProperty1<out Any?, Iterable<T>?>.posOp: KPropertyPath<Any?, T?>
+    get() = colProperty.posOp as KPropertyPath<Any?, T?>
+
+@Suppress("UNCHECKED_CAST")
+fun <T> KProperty1<out Any?, Iterable<T>?>.filteredPosOp(identifier: String): KPropertyPath<Any?, T?> =
+    colProperty.filteredPosOp(identifier) as KPropertyPath<Any?, T?>
+
+suspend fun <T : Any> CoroutineFindPublisher<T>.page(collection: CoroutineCollection<T>, page: Int, pageSize: Int): Page<T> = this
+    .skip(page * pageSize)
+    .limit(pageSize)
+    .toList()
+    .let {
+        Page(it, pageSize, page, ceil(collection.countDocuments() / (pageSize.toDouble())).toInt())
+    }

hub-backend/src/test/kotlin/org/panda_lang/hub/kmongo/KeyProjectionIncrementTest.kt

@@ -32,18 +32,18 @@ fun main(): Unit = runBlocking {
 
     entities.updateOneById(
         "1",
-        inc(IncEntity::map1.keyProjection("key1") / Map1::intMap.keyProjection("key2"), 1),
+        inc(IncEntity::values.keyProjection("key1") / Values::map.keyProjection("key2"), 1),
         upsert()
     )
 }
 
 @Serializable
-data class IncEntity(
+private data class IncEntity(
     val _id: String,
-    val map1: Map<String, Map1> = emptyMap()
+    val values: Map<String, Values> = emptyMap()
 )
 
 @Serializable
-data class Map1(
-    val intMap: Map<String, Int> = emptyMap()
+private data class Values(
+    val map: Map<String, Int> = emptyMap()
 )