From 87393b313adbacd806fd8d56cac0faeb67d32e3f Mon Sep 17 00:00:00 2001 From: Dmitry Marchuk Date: Sun, 11 Aug 2024 21:15:06 +0300 Subject: [PATCH] Add Importer command #dockerpush #latest --- .github/workflows/docker-onpush.yml | 2 +- .gitignore | 1 + build.gradle.kts | 4 +- .../github/smaugfm/monobudget/Application.kt | 57 ++-------- .../io/github/smaugfm/monobudget/Main.kt | 9 +- .../monobudget/common/BaseApplication.kt | 60 +++++++++++ .../common/account/TransferCache.kt | 30 +++++- .../common/account/TransferDetector.kt | 10 +- .../serializer/LocalDateAsISOSerializer.kt | 3 +- .../notify/StatementItemNotificationSender.kt | 11 ++ .../{telegram => notify}/TelegramApi.kt | 2 +- .../TelegramCallbackHandler.kt | 2 +- .../TelegramErrorHandlerEventListener.kt | 2 +- .../TelegramNotificationSender.kt} | 8 +- .../lifecycle/StatementItemProcessor.kt | 6 +- .../transaction/NewTransactionFactory.kt | 2 +- .../import/CsvMonoInstantSerializer.kt | 44 ++++++++ .../smaugfm/monobudget/import/CsvMonoItem.kt | 28 +++++ .../import/ImporterAccountConfig.kt | 6 ++ .../monobudget/import/ImporterApplication.kt | 76 +++++++++++++ .../monobudget/import/ImporterConfig.kt | 29 +++++ .../import/ImporterNotificationSender.kt | 14 +++ .../import/ImporterStatementItem.kt | 30 ++++++ .../import/ImporterStatementSource.kt | 54 ++++++++++ .../import/ImporterTransferCache.kt | 39 +++++++ .../LunchmoneyNewTransactionFactory.kt | 5 +- .../LunchmoneyStatementItemProcessor.kt | 6 +- .../LunchmoneyTelegramCallbackHandler.kt | 2 +- .../lunchmoney/LunchmoneyTransferCache.kt | 5 +- .../lunchmoney/LunchmoneyTransferDetector.kt | 3 +- .../ynab/YnabStatementItemProcessor.kt | 6 +- .../ynab/YnabTelegramCallbackHandler.kt | 2 +- .../monobudget/ynab/YnabTransferCache.kt | 5 +- .../monobudget/ynab/YnabTransferDetector.kt | 3 +- .../common/account/TransferDetectorTest.kt | 6 +- .../monobudget/common/misc/Playground.kt | 100 +++--------------- .../integration/util/IntegrationTestBase.kt | 2 +- 37 files changed, 499 insertions(+), 175 deletions(-) create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/common/BaseApplication.kt create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/common/notify/StatementItemNotificationSender.kt rename src/main/kotlin/io/github/smaugfm/monobudget/common/{telegram => notify}/TelegramApi.kt (98%) rename src/main/kotlin/io/github/smaugfm/monobudget/common/{telegram => notify}/TelegramCallbackHandler.kt (99%) rename src/main/kotlin/io/github/smaugfm/monobudget/common/{telegram => notify}/TelegramErrorHandlerEventListener.kt (98%) rename src/main/kotlin/io/github/smaugfm/monobudget/common/{telegram/TelegramMessageSender.kt => notify/TelegramNotificationSender.kt} (88%) create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/import/CsvMonoInstantSerializer.kt create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/import/CsvMonoItem.kt create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterAccountConfig.kt create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterApplication.kt create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterConfig.kt create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterNotificationSender.kt create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterStatementItem.kt create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterStatementSource.kt create mode 100644 src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterTransferCache.kt diff --git a/.github/workflows/docker-onpush.yml b/.github/workflows/docker-onpush.yml index 75730b2..27bc2ef 100644 --- a/.github/workflows/docker-onpush.yml +++ b/.github/workflows/docker-onpush.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: temurin - java-version: 11 + java-version: 17 cache: gradle - name: docker login run: | diff --git a/.gitignore b/.gitignore index 2c0f73b..61d2cae 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ out/ settings*.json settings*.yml +import-config*.yml retries.json docker-compose.yml diff --git a/build.gradle.kts b/build.gradle.kts index f15f2f9..0e12929 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -42,7 +42,7 @@ dependencies { implementation(kotlin("stdlib")) implementation(kotlin("reflect")) implementation("io.github.smaugfm:monobank:0.0.2") - implementation("io.github.smaugfm:lunchmoney:1.0.3-SNAPSHOT") + implementation("io.github.smaugfm:lunchmoney:1.1.0") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor:$kotlinxCoroutines") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$kotlinxCoroutines") implementation("com.github.livefront.sealed-enum:runtime:$sealedEnum") @@ -54,7 +54,7 @@ dependencies { implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson") implementation("de.brudaswen.kotlinx.serialization:kotlinx-serialization-csv:2.0.0") implementation("com.charleskorn.kaml:kaml:0.55.0") - implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.4.1") + implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.6.0") implementation("io.github.oshai:kotlin-logging:5.1.0") implementation("com.google.code.gson:gson:2.10.1") testImplementation("io.mockk:mockk:1.13.8") diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/Application.kt b/src/main/kotlin/io/github/smaugfm/monobudget/Application.kt index f93ce1c..4846b3a 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/Application.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/Application.kt @@ -1,66 +1,25 @@ package io.github.smaugfm.monobudget import io.github.oshai.kotlinlogging.KotlinLogging -import io.github.smaugfm.monobudget.common.exception.BudgetBackendException +import io.github.smaugfm.monobudget.common.BaseApplication +import io.github.smaugfm.monobudget.common.notify.TelegramApi +import io.github.smaugfm.monobudget.common.notify.TelegramCallbackHandler import io.github.smaugfm.monobudget.common.startup.ApplicationStartupVerifier import io.github.smaugfm.monobudget.common.statement.StatementSource -import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementEvents -import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementItemProcessor -import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingScopeComponent -import io.github.smaugfm.monobudget.common.telegram.TelegramApi -import io.github.smaugfm.monobudget.common.telegram.TelegramCallbackHandler import io.github.smaugfm.monobudget.common.util.injectAll -import kotlinx.coroutines.flow.asFlow -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.filter -import kotlinx.coroutines.flow.flatMapMerge -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.onEach -import org.koin.core.component.KoinComponent import org.koin.core.component.inject import kotlin.system.exitProcess private val log = KotlinLogging.logger {} class Application : - KoinComponent { + BaseApplication() { + override val statementSources: List by injectAll() private val telegramApi by inject() - private val statementSources by injectAll() private val startupVerifiers by injectAll() private val telegramCallbackHandler by inject>() - private val statementEvents by inject() - suspend fun run() { - runStartupChecks() - - statementSources.forEach { it.prepare() } - - telegramApi.start(telegramCallbackHandler::handle) - log.info { "Started application" } - - statementSources.asFlow() - .flatMapMerge { it.statements() } - .filter(statementEvents::onNewStatement) - .map(::StatementProcessingScopeComponent) - .onEach { - with(it) { - try { - scope.get>() - .process() - statementEvents.onStatementEnd(ctx) - } catch (e: BudgetBackendException) { - statementEvents.onStatementRetry(ctx, e) - } catch (e: Throwable) { - statementEvents.onStatementError(ctx, e) - } finally { - scope.close() - } - } - } - .collect() - } - - private suspend fun runStartupChecks() { + override suspend fun beforeStart() { try { startupVerifiers.forEach { it.verify() } } catch (e: Throwable) { @@ -68,4 +27,8 @@ class Application : exitProcess(1) } } + + override suspend fun afterSourcesPrepare() { + telegramApi.start(telegramCallbackHandler::handle) + } } diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/Main.kt b/src/main/kotlin/io/github/smaugfm/monobudget/Main.kt index 4d15d92..59be98a 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/Main.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/Main.kt @@ -14,6 +14,7 @@ import io.github.smaugfm.monobudget.common.model.settings.MonoAccountSettings import io.github.smaugfm.monobudget.common.model.settings.Settings import io.github.smaugfm.monobudget.common.retry.JacksonFileStatementRetryRepository import io.github.smaugfm.monobudget.common.retry.StatementRetryRepository +import io.github.smaugfm.monobudget.import.ImporterApplication import io.github.smaugfm.monobudget.lunchmoney.LunchmoneyModule import io.github.smaugfm.monobudget.mono.MonoApi import io.github.smaugfm.monobudget.mono.MonoModule @@ -37,7 +38,13 @@ private val log = KotlinLogging.logger {} private const val DEFAULT_HTTP_PORT = 80 -fun main() { +fun main(args: Array) { + if (args.size == 1 && args[0] == "importer") { + return runBlocking { + ImporterApplication.main(this) + } + } + val env = System.getenv() val setWebhook = env["SET_WEBHOOK"]?.toBoolean() ?: false val monoWebhookUrl = URI(env["MONO_WEBHOOK_URL"]!!) diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/BaseApplication.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/BaseApplication.kt new file mode 100644 index 0000000..6686f53 --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/BaseApplication.kt @@ -0,0 +1,60 @@ +package io.github.smaugfm.monobudget.common + +import io.github.oshai.kotlinlogging.KotlinLogging +import io.github.smaugfm.monobudget.common.exception.BudgetBackendException +import io.github.smaugfm.monobudget.common.statement.StatementSource +import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementEvents +import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementItemProcessor +import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingScopeComponent +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.flatMapMerge +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.onEach +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject + +private val log = KotlinLogging.logger {} + +abstract class BaseApplication : KoinComponent { + protected abstract val statementSources: List + private val statementEvents by inject() + + open suspend fun run() { + beforeStart() + + statementSources.forEach { it.prepare() } + + afterSourcesPrepare() + + log.info { "Started application" } + + statementSources.asFlow() + .flatMapMerge { it.statements() } + .filter(statementEvents::onNewStatement) + .map(::StatementProcessingScopeComponent) + .onEach { + with(it) { + try { + scope.get>() + .process() + statementEvents.onStatementEnd(ctx) + } catch (e: BudgetBackendException) { + statementEvents.onStatementRetry(ctx, e) + } catch (e: Throwable) { + statementEvents.onStatementError(ctx, e) + } finally { + scope.close() + } + } + } + .collect() + } + + protected open suspend fun beforeStart() { + } + + protected open suspend fun afterSourcesPrepare() { + } +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/account/TransferCache.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/account/TransferCache.kt index 9c8cc17..d764c6c 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/common/account/TransferCache.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/account/TransferCache.kt @@ -3,7 +3,31 @@ package io.github.smaugfm.monobudget.common.account import io.github.smaugfm.monobudget.common.model.financial.StatementItem import io.github.smaugfm.monobudget.common.util.misc.ConcurrentExpiringMap import kotlinx.coroutines.Deferred -import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration -abstract class TransferCache : - ConcurrentExpiringMap>(1.minutes) +interface TransferCache { + suspend fun getEntries(item: StatementItem): Set>> + + suspend fun put( + item: StatementItem, + transaction: Deferred, + ) + + open class Expiring(expirationDuration: Duration) : + TransferCache { + private val cache = ConcurrentExpiringMap>(expirationDuration) + + override suspend fun getEntries( + item: StatementItem, + ): Set>> { + return cache.entries + } + + override suspend fun put( + item: StatementItem, + transaction: Deferred, + ) { + cache.add(item, transaction) + } + } +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/account/TransferDetector.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/account/TransferDetector.kt index abdbecb..6a75c02 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/common/account/TransferDetector.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/account/TransferDetector.kt @@ -3,26 +3,24 @@ package io.github.smaugfm.monobudget.common.account import io.github.oshai.kotlinlogging.KotlinLogging import io.github.smaugfm.monobudget.common.model.financial.StatementItem import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingContext -import io.github.smaugfm.monobudget.common.util.misc.ConcurrentExpiringMap import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Deferred private val log = KotlinLogging.logger {} abstract class TransferDetector( private val bankAccounts: BankAccountService, private val ctx: StatementProcessingContext, - private val cache: ConcurrentExpiringMap>, + private val cache: TransferCache, ) { suspend fun checkForTransfer(): MaybeTransfer = ctx.getOrPut("transfer") { val existingTransfer = - cache.entries.firstOrNull { (recentStatementItem) -> + cache.getEntries(ctx.item).firstOrNull { (recentStatementItem) -> checkIsTransferTransactions(recentStatementItem) }?.value?.await() if (existingTransfer != null) { - log.debug { + log.info { "Found matching transfer transaction.\n" + "Current: ${ctx.item}\n" + "Recent transfer: $existingTransfer" @@ -30,7 +28,7 @@ abstract class TransferDetector( MaybeTransfer.Transfer(ctx.item, existingTransfer) } else { val deferred = CompletableDeferred() - cache.add(ctx.item, deferred) + cache.put(ctx.item, deferred) MaybeTransfer.NotTransfer(ctx.item, deferred) } diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/model/serializer/LocalDateAsISOSerializer.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/model/serializer/LocalDateAsISOSerializer.kt index 8914a7f..e42a703 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/common/model/serializer/LocalDateAsISOSerializer.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/model/serializer/LocalDateAsISOSerializer.kt @@ -1,7 +1,6 @@ package io.github.smaugfm.monobudget.common.model.serializer import kotlinx.datetime.LocalDate -import kotlinx.datetime.toLocalDate import kotlinx.serialization.KSerializer import kotlinx.serialization.descriptors.PrimitiveKind import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor @@ -23,6 +22,6 @@ class LocalDateAsISOSerializer : KSerializer { } override fun deserialize(decoder: Decoder): LocalDate { - return decoder.decodeString().toLocalDate() + return LocalDate.parse(decoder.decodeString()) } } diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/StatementItemNotificationSender.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/StatementItemNotificationSender.kt new file mode 100644 index 0000000..3fcdb26 --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/StatementItemNotificationSender.kt @@ -0,0 +1,11 @@ +package io.github.smaugfm.monobudget.common.notify + +import io.github.smaugfm.monobudget.common.model.financial.BankAccountId +import io.github.smaugfm.monobudget.common.model.telegram.MessageWithReplyKeyboard + +interface StatementItemNotificationSender { + suspend fun notify( + accountId: BankAccountId, + newMessage: MessageWithReplyKeyboard, + ) +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramApi.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramApi.kt similarity index 98% rename from src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramApi.kt rename to src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramApi.kt index 58096b2..3148348 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramApi.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramApi.kt @@ -1,4 +1,4 @@ -package io.github.smaugfm.monobudget.common.telegram +package io.github.smaugfm.monobudget.common.notify import com.elbekd.bot.Bot import com.elbekd.bot.model.ChatId diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramCallbackHandler.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramCallbackHandler.kt similarity index 99% rename from src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramCallbackHandler.kt rename to src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramCallbackHandler.kt index e577c46..4322259 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramCallbackHandler.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramCallbackHandler.kt @@ -1,4 +1,4 @@ -package io.github.smaugfm.monobudget.common.telegram +package io.github.smaugfm.monobudget.common.notify import com.elbekd.bot.model.ChatId import com.elbekd.bot.model.TelegramApiError diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramErrorHandlerEventListener.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramErrorHandlerEventListener.kt similarity index 98% rename from src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramErrorHandlerEventListener.kt rename to src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramErrorHandlerEventListener.kt index 1747373..dcf406a 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramErrorHandlerEventListener.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramErrorHandlerEventListener.kt @@ -1,4 +1,4 @@ -package io.github.smaugfm.monobudget.common.telegram +package io.github.smaugfm.monobudget.common.notify import com.elbekd.bot.model.ChatId import com.elbekd.bot.types.CallbackQuery diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramMessageSender.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramNotificationSender.kt similarity index 88% rename from src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramMessageSender.kt rename to src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramNotificationSender.kt index 99463af..0101f57 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/common/telegram/TelegramMessageSender.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/notify/TelegramNotificationSender.kt @@ -1,4 +1,4 @@ -package io.github.smaugfm.monobudget.common.telegram +package io.github.smaugfm.monobudget.common.notify import com.elbekd.bot.model.ChatId import com.elbekd.bot.types.ParseMode @@ -11,11 +11,11 @@ import org.koin.core.annotation.Single private val log = KotlinLogging.logger {} @Single -class TelegramMessageSender( +class TelegramNotificationSender( private val bankAccounts: BankAccountService, private val telegramApi: TelegramApi, -) { - suspend fun send( +) : StatementItemNotificationSender { + override suspend fun notify( accountId: BankAccountId, newMessage: MessageWithReplyKeyboard, ) { diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/statement/lifecycle/StatementItemProcessor.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/statement/lifecycle/StatementItemProcessor.kt index 25c1902..7aebdfd 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/common/statement/lifecycle/StatementItemProcessor.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/statement/lifecycle/StatementItemProcessor.kt @@ -3,7 +3,7 @@ package io.github.smaugfm.monobudget.common.statement.lifecycle import io.github.oshai.kotlinlogging.KotlinLogging import io.github.smaugfm.monobudget.common.account.BankAccountService import io.github.smaugfm.monobudget.common.account.TransferDetector -import io.github.smaugfm.monobudget.common.telegram.TelegramMessageSender +import io.github.smaugfm.monobudget.common.notify.StatementItemNotificationSender import io.github.smaugfm.monobudget.common.transaction.TransactionFactory import io.github.smaugfm.monobudget.common.transaction.TransactionMessageFormatter import io.github.smaugfm.monobudget.common.util.pp @@ -16,7 +16,7 @@ abstract class StatementItemProcessor( private val bankAccounts: BankAccountService, private val transferDetector: TransferDetector, private val messageFormatter: TransactionMessageFormatter, - private val telegramMessageSender: TelegramMessageSender, + private val notificationSender: StatementItemNotificationSender, ) { suspend fun process() { logStatement() @@ -30,7 +30,7 @@ abstract class StatementItemProcessor( val transaction = transactionFactory.create(maybeTransfer) val message = messageFormatter.format(ctx.item, transaction) - telegramMessageSender.send(ctx.item.accountId, message) + notificationSender.notify(ctx.item.accountId, message) } private suspend fun logStatement() { diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/common/transaction/NewTransactionFactory.kt b/src/main/kotlin/io/github/smaugfm/monobudget/common/transaction/NewTransactionFactory.kt index c733b2a..95ae86e 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/common/transaction/NewTransactionFactory.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/common/transaction/NewTransactionFactory.kt @@ -24,6 +24,6 @@ abstract class NewTransactionFactory : KoinComponent { companion object { @JvmStatic - protected fun StatementItem?.formatDescription() = (this?.description ?: "").replaceNewLines() + fun StatementItem?.formatDescription() = (this?.description ?: "").replaceNewLines() } } diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/import/CsvMonoInstantSerializer.kt b/src/main/kotlin/io/github/smaugfm/monobudget/import/CsvMonoInstantSerializer.kt new file mode 100644 index 0000000..5289197 --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/import/CsvMonoInstantSerializer.kt @@ -0,0 +1,44 @@ +package io.github.smaugfm.monobudget.import + +import kotlinx.datetime.Instant +import kotlinx.datetime.LocalDate +import kotlinx.datetime.LocalTime +import kotlinx.datetime.TimeZone +import kotlinx.datetime.atDate +import kotlinx.datetime.format.char +import kotlinx.datetime.toInstant +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder + +class CsvMonoInstantSerializer : KSerializer { + override val descriptor = + PrimitiveSerialDescriptor(this::class.qualifiedName!!, PrimitiveKind.STRING) + + private val dateFormat = + LocalDate.Format { + dayOfMonth() + char('.') + monthNumber() + char('.') + year() + } + + override fun deserialize(decoder: Decoder): Instant { + val str = decoder.decodeString() + + val localdate = LocalDate.parse(str.substringBefore(" "), dateFormat) + val localtime = LocalTime.parse(str.substringAfter(" ")) + + return localtime.atDate(localdate).toInstant(TimeZone.currentSystemDefault()) + } + + override fun serialize( + encoder: Encoder, + value: Instant, + ) { + TODO("Not yet implemented") + } +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/import/CsvMonoItem.kt b/src/main/kotlin/io/github/smaugfm/monobudget/import/CsvMonoItem.kt new file mode 100644 index 0000000..f023c86 --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/import/CsvMonoItem.kt @@ -0,0 +1,28 @@ +package io.github.smaugfm.monobudget.import + +import io.github.smaugfm.monobudget.common.model.financial.BankAccountId +import io.github.smaugfm.monobudget.common.model.serializer.CurrencyAsStringSerializer +import kotlinx.datetime.Instant +import kotlinx.serialization.Serializable +import java.util.Currency + +@Serializable +internal data class CsvMonoItem( + @Serializable(CsvMonoInstantSerializer::class) + val date: Instant, + val description: String, + val mcc: Int, + val cardCurrencyAmount: Double, + val transactionCurrencyAmount: Double, + @Serializable(CurrencyAsStringSerializer::class) + val currency: Currency, + val exchangeRate: Double?, + val cardCurrencyCommissionAmount: Double?, + val cardCurrencyCashbackAmount: Double?, + val balance: Double?, +) { + fun toStatementItem( + accountId: BankAccountId, + accountCurrency: Currency, + ) = ImporterStatementItem(this, accountId, accountCurrency) +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterAccountConfig.kt b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterAccountConfig.kt new file mode 100644 index 0000000..64a3f5c --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterAccountConfig.kt @@ -0,0 +1,6 @@ +package io.github.smaugfm.monobudget.import + +data class ImporterAccountConfig( + val accountAlias: String, + val transactionsFileContent: String, +) diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterApplication.kt b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterApplication.kt new file mode 100644 index 0000000..77e1811 --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterApplication.kt @@ -0,0 +1,76 @@ +package io.github.smaugfm.monobudget.import + +import io.github.oshai.kotlinlogging.KotlinLogging +import io.github.smaugfm.lunchmoney.model.LunchmoneyInsertTransaction +import io.github.smaugfm.lunchmoney.model.LunchmoneyTransaction +import io.github.smaugfm.monobudget.common.BaseApplication +import io.github.smaugfm.monobudget.common.account.TransferCache +import io.github.smaugfm.monobudget.common.model.settings.Settings +import io.github.smaugfm.monobudget.common.notify.StatementItemNotificationSender +import io.github.smaugfm.monobudget.common.retry.InMemoryStatementRetryRepository +import io.github.smaugfm.monobudget.common.transaction.NewTransactionFactory +import io.github.smaugfm.monobudget.lunchmoney.LunchmoneyNewTransactionFactory +import io.github.smaugfm.monobudget.mono.MonoWebhookSettings +import io.github.smaugfm.monobudget.setupKoinModules +import kotlinx.coroutines.CoroutineScope +import kotlinx.datetime.Clock +import org.koin.core.context.startKoin +import org.koin.dsl.module +import java.net.URI +import java.nio.file.Paths +import kotlin.system.exitProcess +import kotlin.time.Duration.Companion.seconds + +private val log = KotlinLogging.logger {} + +class ImporterApplication(private val source: ImporterStatementSource) : + BaseApplication() { + override val statementSources = listOf(source) + + override suspend fun run() { + super.run() + log.info { "Import finished" } + exitProcess(0) + } + + companion object { + suspend fun main(coroutineScope: CoroutineScope) { + val settings = + Settings.load( + Paths.get(System.getenv()["SETTINGS_FILE"] ?: "settings.yml"), + ) + val importConfig = + ImporterConfig.load( + Paths.get(System.getenv()["IMPORT_CONFIG_FILE"] ?: "import-config.yml"), + ) + val noteSuffix = " monobudget-import-${Clock.System.now()}" + log.info { + "Importing transactions from files: \n${ + importConfig.getImports().map { it.transactionsFileContent }.joinToString("\n") + }\nUsing 'note' field suffix: '${noteSuffix.trim()}'" + } + + startKoin { + setupKoinModules( + coroutineScope, + InMemoryStatementRetryRepository(), + settings, + MonoWebhookSettings(false, URI.create("none://none"), 0), + ) + modules( + module { + single> { + LunchmoneyNewTransactionFactory(noteSuffix) + } + single> { + ImporterTransferCache(7.seconds) + } + single { ImporterNotificationSender } + }, + ) + }.koin + + ImporterApplication(ImporterStatementSource(importConfig.getImports())).run() + } + } +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterConfig.kt b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterConfig.kt new file mode 100644 index 0000000..2627a9f --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterConfig.kt @@ -0,0 +1,29 @@ +package io.github.smaugfm.monobudget.import + +import com.charleskorn.kaml.Yaml +import com.charleskorn.kaml.YamlConfiguration +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import java.io.File +import java.nio.file.Path + +private val log = KotlinLogging.logger {} + +@Serializable +data class ImporterConfig( + private val imports: Map, +) { + fun getImports() = imports.entries.map { ImporterAccountConfig(it.key, it.value) } + + companion object { + fun load(path: Path): ImporterConfig = load(File(path.toString()).readText()) + + private fun load(content: String): ImporterConfig = + Yaml(configuration = YamlConfiguration(strictMode = false)) + .decodeFromString(content) + .also { + log.debug { "Loaded import-config: $it" } + } + } +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterNotificationSender.kt b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterNotificationSender.kt new file mode 100644 index 0000000..67da272 --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterNotificationSender.kt @@ -0,0 +1,14 @@ +package io.github.smaugfm.monobudget.import + +import io.github.smaugfm.monobudget.common.model.financial.BankAccountId +import io.github.smaugfm.monobudget.common.model.telegram.MessageWithReplyKeyboard +import io.github.smaugfm.monobudget.common.notify.StatementItemNotificationSender + +object ImporterNotificationSender : StatementItemNotificationSender { + override suspend fun notify( + accountId: BankAccountId, + newMessage: MessageWithReplyKeyboard, + ) { + // no-op + } +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterStatementItem.kt b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterStatementItem.kt new file mode 100644 index 0000000..6aefce1 --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterStatementItem.kt @@ -0,0 +1,30 @@ +package io.github.smaugfm.monobudget.import + +import io.github.smaugfm.monobudget.common.model.financial.Amount +import io.github.smaugfm.monobudget.common.model.financial.BankAccountId +import io.github.smaugfm.monobudget.common.model.financial.StatementItem +import java.util.Currency +import java.util.UUID + +internal data class ImporterStatementItem( + val csv: CsvMonoItem, + override val accountId: BankAccountId, + val accountCurrency: Currency, +) : StatementItem { + override val id = UUID.randomUUID().toString() + override val time = csv.date + override val description = csv.description + override val comment = null + override val mcc = csv.mcc + override val amount = + Amount.fromLunchmoneyAmount( + csv.cardCurrencyAmount, + accountCurrency, + ) + override val operationAmount = + Amount.fromLunchmoneyAmount( + csv.transactionCurrencyAmount, + csv.currency, + ) + override val currency = csv.currency +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterStatementSource.kt b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterStatementSource.kt new file mode 100644 index 0000000..a8d5627 --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterStatementSource.kt @@ -0,0 +1,54 @@ +package io.github.smaugfm.monobudget.import + +import io.github.oshai.kotlinlogging.KotlinLogging +import io.github.smaugfm.monobudget.common.account.BankAccountService +import io.github.smaugfm.monobudget.common.model.financial.StatementItem +import io.github.smaugfm.monobudget.common.statement.StatementSource +import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingContext +import io.github.smaugfm.monobudget.mono.MonoApi +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.asFlow +import kotlinx.serialization.csv.Csv +import kotlinx.serialization.decodeFromString +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject +import org.koin.core.qualifier.StringQualifier +import java.io.File + +private val log = KotlinLogging.logger {} + +class ImporterStatementSource(private val configs: List) : + StatementSource, KoinComponent { + lateinit var statementItems: List + + private val bankAccountsService: BankAccountService by inject() + private val csv = + Csv { + hasHeaderRecord = false + nullString = "—" + } + + override suspend fun prepare() { + statementItems = + configs.map { config -> + val accountId = + getKoin().get(StringQualifier(config.accountAlias)) + .accountId + val accountCurrency = bankAccountsService.getAccountCurrency(accountId)!! + val csvItems = + csv.decodeFromString>( + File(config.transactionsFileContent).readText() + .substringAfter("\n"), + ) + + log.info { "Found ${csvItems.size} transactions for '${config.accountAlias}' account" } + csvItems.map { it.toStatementItem(accountId, accountCurrency) } + }.flatten() + .sortedBy { it.time } + log.info { "Importing total of ${statementItems.size} transactions" } + } + + override suspend fun statements(): Flow { + return statementItems.map(::StatementProcessingContext).asFlow() + } +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterTransferCache.kt b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterTransferCache.kt new file mode 100644 index 0000000..255c5ec --- /dev/null +++ b/src/main/kotlin/io/github/smaugfm/monobudget/import/ImporterTransferCache.kt @@ -0,0 +1,39 @@ +package io.github.smaugfm.monobudget.import + +import io.github.smaugfm.lunchmoney.model.LunchmoneyTransaction +import io.github.smaugfm.monobudget.common.account.TransferCache +import io.github.smaugfm.monobudget.common.model.financial.StatementItem +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.newSingleThreadContext +import kotlinx.coroutines.withContext +import kotlin.time.Duration + +@OptIn(DelicateCoroutinesApi::class) +class ImporterTransferCache(private val cacheTimeSimulatedDuration: Duration) : + TransferCache { + private val singleThreadedContext = newSingleThreadContext("import-transfer-cache") + private val cache = + mutableMapOf>() + .toSortedMap { o1, o2 -> o1.time.compareTo(o2.time) } + + @Suppress("DeferredResultUnused") + override suspend fun getEntries( + item: StatementItem, + ): Set>> = + withContext(singleThreadedContext) { + val threshold = item.time - cacheTimeSimulatedDuration + for (key in cache.keys.filter { it.time < threshold }) { + cache.remove(key) + } + cache.entries + } + + override suspend fun put( + item: StatementItem, + transaction: Deferred, + ) { + check(cache.keys.all { item.time >= it.time }) + cache[item] = transaction + } +} diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyNewTransactionFactory.kt b/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyNewTransactionFactory.kt index fe461cf..ddf1832 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyNewTransactionFactory.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyNewTransactionFactory.kt @@ -13,7 +13,8 @@ import java.util.Currency private val log = KotlinLogging.logger {} @Single -class LunchmoneyNewTransactionFactory : NewTransactionFactory() { +class LunchmoneyNewTransactionFactory(private val noteSuffix: String = "") : + NewTransactionFactory() { override suspend fun create(statement: StatementItem): LunchmoneyInsertTransaction { log.debug { "Transforming Monobank statement to Lunchmoney transaction." } @@ -28,7 +29,7 @@ class LunchmoneyNewTransactionFactory : NewTransactionFactory, messageFormatter: TransactionMessageFormatter, - telegramMessageSender: TelegramMessageSender, + notificationSender: StatementItemNotificationSender, ) : StatementItemProcessor( ctx, transactionFactory, bankAccounts, transferDetector, messageFormatter, - telegramMessageSender, + notificationSender, ) diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTelegramCallbackHandler.kt b/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTelegramCallbackHandler.kt index fe718dd..04bd943 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTelegramCallbackHandler.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTelegramCallbackHandler.kt @@ -8,7 +8,7 @@ import io.github.smaugfm.lunchmoney.model.LunchmoneyUpdateTransaction import io.github.smaugfm.lunchmoney.model.enumeration.LunchmoneyTransactionStatus import io.github.smaugfm.monobudget.common.category.CategoryService import io.github.smaugfm.monobudget.common.model.callback.TransactionUpdateType -import io.github.smaugfm.monobudget.common.telegram.TelegramCallbackHandler +import io.github.smaugfm.monobudget.common.notify.TelegramCallbackHandler import io.github.smaugfm.monobudget.common.transaction.TransactionMessageFormatter.Companion.extractFromOldMessage import io.github.smaugfm.monobudget.common.transaction.TransactionMessageFormatter.Companion.formatHTMLStatementMessage import io.github.smaugfm.monobudget.lunchmoney.LunchmoneyTransactionMessageFormatter.Companion.constructTransactionsQuickUrl diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTransferCache.kt b/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTransferCache.kt index 1dfed23..469a9c1 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTransferCache.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTransferCache.kt @@ -3,6 +3,9 @@ package io.github.smaugfm.monobudget.lunchmoney import io.github.smaugfm.lunchmoney.model.LunchmoneyTransaction import io.github.smaugfm.monobudget.common.account.TransferCache import org.koin.core.annotation.Single +import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes @Single -class LunchmoneyTransferCache : TransferCache() +class LunchmoneyTransferCache(expirationDuration: Duration = 1.minutes) : + TransferCache.Expiring(expirationDuration) diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTransferDetector.kt b/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTransferDetector.kt index 5d91d9d..a0bb8d7 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTransferDetector.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/lunchmoney/LunchmoneyTransferDetector.kt @@ -2,6 +2,7 @@ package io.github.smaugfm.monobudget.lunchmoney import io.github.smaugfm.lunchmoney.model.LunchmoneyTransaction import io.github.smaugfm.monobudget.common.account.BankAccountService +import io.github.smaugfm.monobudget.common.account.TransferCache import io.github.smaugfm.monobudget.common.account.TransferDetector import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingContext import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingScopeComponent @@ -13,5 +14,5 @@ import org.koin.core.annotation.Scoped class LunchmoneyTransferDetector( bankAccounts: BankAccountService, ctx: StatementProcessingContext, - cache: LunchmoneyTransferCache, + cache: TransferCache, ) : TransferDetector(bankAccounts, ctx, cache) diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabStatementItemProcessor.kt b/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabStatementItemProcessor.kt index b0f619e..45fddad 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabStatementItemProcessor.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabStatementItemProcessor.kt @@ -2,10 +2,10 @@ package io.github.smaugfm.monobudget.ynab import io.github.smaugfm.monobudget.common.account.BankAccountService import io.github.smaugfm.monobudget.common.account.TransferDetector +import io.github.smaugfm.monobudget.common.notify.StatementItemNotificationSender import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementItemProcessor import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingContext import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingScopeComponent -import io.github.smaugfm.monobudget.common.telegram.TelegramMessageSender import io.github.smaugfm.monobudget.common.transaction.TransactionFactory import io.github.smaugfm.monobudget.common.transaction.TransactionMessageFormatter import io.github.smaugfm.monobudget.ynab.model.YnabSaveTransaction @@ -21,12 +21,12 @@ class YnabStatementItemProcessor( bankAccounts: BankAccountService, transferDetector: TransferDetector, messageFormatter: TransactionMessageFormatter, - telegramMessageSender: TelegramMessageSender, + notificationSender: StatementItemNotificationSender, ) : StatementItemProcessor( ctx, transactionFactory, bankAccounts, transferDetector, messageFormatter, - telegramMessageSender, + notificationSender, ) diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTelegramCallbackHandler.kt b/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTelegramCallbackHandler.kt index 70031e4..99d6809 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTelegramCallbackHandler.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTelegramCallbackHandler.kt @@ -2,7 +2,7 @@ package io.github.smaugfm.monobudget.ynab import com.elbekd.bot.types.Message import io.github.smaugfm.monobudget.common.model.callback.TransactionUpdateType -import io.github.smaugfm.monobudget.common.telegram.TelegramCallbackHandler +import io.github.smaugfm.monobudget.common.notify.TelegramCallbackHandler import io.github.smaugfm.monobudget.common.transaction.TransactionMessageFormatter.Companion.extractFromOldMessage import io.github.smaugfm.monobudget.common.transaction.TransactionMessageFormatter.Companion.formatHTMLStatementMessage import io.github.smaugfm.monobudget.ynab.model.YnabTransactionDetail diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTransferCache.kt b/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTransferCache.kt index 46bcdaa..ad62c92 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTransferCache.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTransferCache.kt @@ -3,6 +3,9 @@ package io.github.smaugfm.monobudget.ynab import io.github.smaugfm.monobudget.common.account.TransferCache import io.github.smaugfm.monobudget.ynab.model.YnabTransactionDetail import org.koin.core.annotation.Single +import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes @Single -class YnabTransferCache : TransferCache() +class YnabTransferCache(expirationDuration: Duration = 1.minutes) : + TransferCache.Expiring(expirationDuration) diff --git a/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTransferDetector.kt b/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTransferDetector.kt index 55ed435..23dbed8 100644 --- a/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTransferDetector.kt +++ b/src/main/kotlin/io/github/smaugfm/monobudget/ynab/YnabTransferDetector.kt @@ -1,6 +1,7 @@ package io.github.smaugfm.monobudget.ynab import io.github.smaugfm.monobudget.common.account.BankAccountService +import io.github.smaugfm.monobudget.common.account.TransferCache import io.github.smaugfm.monobudget.common.account.TransferDetector import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingContext import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingScopeComponent @@ -13,7 +14,7 @@ import org.koin.core.annotation.Scoped class YnabTransferDetector( bankAccounts: BankAccountService, ctx: StatementProcessingContext, - cache: YnabTransferCache, + cache: TransferCache, ) : TransferDetector( bankAccounts, ctx, diff --git a/src/test/kotlin/io/github/smaugfm/monobudget/common/account/TransferDetectorTest.kt b/src/test/kotlin/io/github/smaugfm/monobudget/common/account/TransferDetectorTest.kt index e4cdbaa..990fe35 100644 --- a/src/test/kotlin/io/github/smaugfm/monobudget/common/account/TransferDetectorTest.kt +++ b/src/test/kotlin/io/github/smaugfm/monobudget/common/account/TransferDetectorTest.kt @@ -8,14 +8,11 @@ import io.github.smaugfm.monobank.model.MonoWebhookResponseData import io.github.smaugfm.monobudget.TestBase import io.github.smaugfm.monobudget.common.account.MaybeTransfer.NotTransfer import io.github.smaugfm.monobudget.common.account.MaybeTransfer.Transfer -import io.github.smaugfm.monobudget.common.model.financial.StatementItem import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingContext import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingScopeComponent -import io.github.smaugfm.monobudget.common.util.misc.ConcurrentExpiringMap import io.github.smaugfm.monobudget.mono.MonobankWebhookResponseStatementItem import io.mockk.coEvery import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Deferred import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking @@ -33,7 +30,8 @@ private val log = KotlinLogging.logger { } class TransferDetectorTest : TestBase() { companion object { - private val cache = ConcurrentExpiringMap>(1.minutes) + private val cache = + TransferCache.Expiring(1.minutes) } class TestDetector( diff --git a/src/test/kotlin/io/github/smaugfm/monobudget/common/misc/Playground.kt b/src/test/kotlin/io/github/smaugfm/monobudget/common/misc/Playground.kt index 58921cb..c7d2363 100644 --- a/src/test/kotlin/io/github/smaugfm/monobudget/common/misc/Playground.kt +++ b/src/test/kotlin/io/github/smaugfm/monobudget/common/misc/Playground.kt @@ -2,34 +2,28 @@ package io.github.smaugfm.monobudget.common.misc import io.github.smaugfm.lunchmoney.api.LunchmoneyApi import io.github.smaugfm.monobudget.common.model.BudgetBackend +import io.github.smaugfm.monobudget.common.model.settings.MonoAccountSettings import io.github.smaugfm.monobudget.common.model.settings.Settings -import io.github.smaugfm.monobudget.common.util.MCCRegistry +import io.github.smaugfm.monobudget.common.util.injectAll import io.github.smaugfm.monobudget.common.util.misc.PeriodicFetcherFactory import io.github.smaugfm.monobudget.lunchmoney.LunchmoneyCategoryService +import io.github.smaugfm.monobudget.mono.MonoApi import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.runBlocking -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import kotlinx.serialization.csv.Csv -import kotlinx.serialization.decodeFromString -import kotlinx.serialization.encodeToString import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import org.koin.core.context.startKoin +import org.koin.core.qualifier.StringQualifier import org.koin.dsl.bind import org.koin.dsl.module import org.koin.test.KoinTest -import org.koin.test.inject import java.nio.file.Paths import kotlin.io.path.readText -@Disabled @OptIn(DelicateCoroutinesApi::class) class Playground : KoinTest { - private val categorySuggestion: LunchmoneyCategoryService by inject() + private val apis: List by injectAll() companion object { @BeforeAll @@ -45,6 +39,16 @@ class Playground : KoinTest { single { settings.mcc } single { LunchmoneyApi(settings.budgetBackend.token) } single { LunchmoneyCategoryService(get(), get()) } + settings.accounts.settings.filterIsInstance() + .forEach { s -> + single(StringQualifier(s.alias)) { + MonoApi( + s.token, + s.accountId, + s.alias, + ) + } + } }, ) } @@ -52,77 +56,7 @@ class Playground : KoinTest { } @Test - @Disabled - fun test() { - runBlocking { - val csv = - Csv { - hasHeaderRecord = true - } - - println() - println() - println() - - val output = - csv.decodeFromString>( - Paths.get("/Users/smaugfm/Downloads/report_26-03-2023_11-30-38.csv") - .toFile().readText(), - ).map { - CsvOutputItem( - it.time, - categorySuggestion.inferCategoryNameByMcc(it.mcc.toInt()) ?: "", - it.details, - if (it.currency == "UAH") { - it.amount + it.currency - } else { - it.operationAmount + it.currency - }, - "${it.mcc} " + MCCRegistry.map[it.mcc.toInt()]?.fullDescription, - ) - } - - Paths.get("/Users/smaugfm/Downloads/output.csv").toFile().writeText( - csv.encodeToString(output), - ) - } + fun vasa() { + println(apis) } - - @Serializable - data class CsvOutputItem( - @SerialName("Дата i час операції") - val time: String, - @SerialName("Запропонована категорія") - val suggestedCategory: String, - @SerialName("Деталі операції") - val details: String, - @SerialName("Сума операції") - val amount: String, - @SerialName("MCC деталі") - val mcc: String, - ) - - @Serializable - data class CsvMonoItem( - @SerialName("Дата i час операції") - val time: String, - @SerialName("Деталі операції") - val details: String, - @SerialName("MCC") - val mcc: String, - @SerialName("Сума в валюті картки (UAH)") - val amount: String, - @SerialName("Сума в валюті операції") - val operationAmount: String, - @SerialName("Валюта") - val currency: String, - @SerialName("Курс") - val exchangeRate: String, - @SerialName("Сума комісій (UAH)") - val commissionAmount: String, - @SerialName("Сума кешбеку (UAH)") - val cashbackAmount: String, - @SerialName("Залишок після операції") - val balance: String, - ) } diff --git a/src/test/kotlin/io/github/smaugfm/monobudget/integration/util/IntegrationTestBase.kt b/src/test/kotlin/io/github/smaugfm/monobudget/integration/util/IntegrationTestBase.kt index ad52fb1..8e25a68 100644 --- a/src/test/kotlin/io/github/smaugfm/monobudget/integration/util/IntegrationTestBase.kt +++ b/src/test/kotlin/io/github/smaugfm/monobudget/integration/util/IntegrationTestBase.kt @@ -10,13 +10,13 @@ import io.github.smaugfm.monobudget.Application import io.github.smaugfm.monobudget.TestBase import io.github.smaugfm.monobudget.TestData import io.github.smaugfm.monobudget.common.model.settings.Settings +import io.github.smaugfm.monobudget.common.notify.TelegramApi import io.github.smaugfm.monobudget.common.retry.InMemoryStatementRetryRepository import io.github.smaugfm.monobudget.common.retry.StatementRetryRepository import io.github.smaugfm.monobudget.common.startup.ApplicationStartupVerifier import io.github.smaugfm.monobudget.common.startup.BudgetSettingsVerifier import io.github.smaugfm.monobudget.common.statement.StatementSource import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingContext -import io.github.smaugfm.monobudget.common.telegram.TelegramApi import io.github.smaugfm.monobudget.common.util.misc.PeriodicFetcherFactory import io.github.smaugfm.monobudget.integration.TransactionsTest import io.github.smaugfm.monobudget.mono.MonoWebhookListener