monobudget import: WIP

.gitignore

@@ -7,5 +7,6 @@ out/
 
 settings*.json
 settings*.yml
+import-config*.yml
 retries.json
 docker-compose.yml

build.gradle.kts

@@ -54,7 +54,7 @@ dependencies {
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson")
     implementation("de.brudaswen.kotlinx.serialization:kotlinx-serialization-csv:2.0.0")
     implementation("com.charleskorn.kaml:kaml:0.55.0")
-    implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.4.1")
+    implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.6.0")
     implementation("io.github.oshai:kotlin-logging:5.1.0")
     implementation("com.google.code.gson:gson:2.10.1")
     testImplementation("io.mockk:mockk:1.13.8")

src/main/kotlin/io/github/smaugfm/monobudget/Application.kt

@@ -1,71 +1,33 @@
 package io.github.smaugfm.monobudget
 
 import io.github.oshai.kotlinlogging.KotlinLogging
-import io.github.smaugfm.monobudget.common.exception.BudgetBackendException
+import io.github.smaugfm.monobudget.common.BaseApplication
 import io.github.smaugfm.monobudget.common.startup.ApplicationStartupVerifier
 import io.github.smaugfm.monobudget.common.statement.StatementSource
-import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementEvents
-import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementItemProcessor
-import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingScopeComponent
 import io.github.smaugfm.monobudget.common.telegram.TelegramApi
 import io.github.smaugfm.monobudget.common.telegram.TelegramCallbackHandler
 import io.github.smaugfm.monobudget.common.util.injectAll
-import kotlinx.coroutines.flow.asFlow
-import kotlinx.coroutines.flow.collect
-import kotlinx.coroutines.flow.filter
-import kotlinx.coroutines.flow.flatMapMerge
-import kotlinx.coroutines.flow.map
-import kotlinx.coroutines.flow.onEach
-import org.koin.core.component.KoinComponent
 import org.koin.core.component.inject
 import kotlin.system.exitProcess
 
 private val log = KotlinLogging.logger {}
 
-class Application<TTransaction, TNewTransaction> :
-    KoinComponent {
+class Application<TTransaction, TNewTransaction>(statementSources: List<StatementSource>) :
+    BaseApplication<TTransaction, TNewTransaction>(statementSources) {
     private val telegramApi by inject<TelegramApi>()
-    private val statementSources by injectAll<StatementSource>()
     private val startupVerifiers by injectAll<ApplicationStartupVerifier>()
     private val telegramCallbackHandler by inject<TelegramCallbackHandler<TTransaction>>()
-    private val statementEvents by inject<StatementEvents>()
 
-    suspend fun run() {
-        runStartupChecks()
-
-        statementSources.forEach { it.prepare() }
-
-        telegramApi.start(telegramCallbackHandler::handle)
-        log.info { "Started application" }
-
-        statementSources.asFlow()
-            .flatMapMerge { it.statements() }
-            .filter(statementEvents::onNewStatement)
-            .map(::StatementProcessingScopeComponent)
-            .onEach {
-                with(it) {
-                    try {
-                        scope.get<StatementItemProcessor<TTransaction, TNewTransaction>>()
-                            .process()
-                        statementEvents.onStatementEnd(ctx)
-                    } catch (e: BudgetBackendException) {
-                        statementEvents.onStatementRetry(ctx, e)
-                    } catch (e: Throwable) {
-                        statementEvents.onStatementError(ctx, e)
-                    } finally {
-                        scope.close()
-                    }
-                }
-            }
-            .collect()
-    }
-
-    private suspend fun runStartupChecks() {
+    override suspend fun beforeStart() {
         try {
             startupVerifiers.forEach { it.verify() }
         } catch (e: Throwable) {
             log.error(e) { "Failed to start application. Exiting..." }
             exitProcess(1)
         }
     }
+
+    override suspend fun afterSourcesPrepare() {
+        telegramApi.start(telegramCallbackHandler::handle)
+    }
 }

src/main/kotlin/io/github/smaugfm/monobudget/Main.kt

@@ -14,6 +14,7 @@ import io.github.smaugfm.monobudget.common.model.settings.MonoAccountSettings
 import io.github.smaugfm.monobudget.common.model.settings.Settings
 import io.github.smaugfm.monobudget.common.retry.JacksonFileStatementRetryRepository
 import io.github.smaugfm.monobudget.common.retry.StatementRetryRepository
+import io.github.smaugfm.monobudget.importer.ImportApplication
 import io.github.smaugfm.monobudget.lunchmoney.LunchmoneyModule
 import io.github.smaugfm.monobudget.mono.MonoApi
 import io.github.smaugfm.monobudget.mono.MonoModule
@@ -37,7 +38,13 @@ private val log = KotlinLogging.logger {}
 
 private const val DEFAULT_HTTP_PORT = 80
 
-fun main() {
+fun main(args: Array<String>) {
+    if (args.size == 1 && args[0] == "import") {
+        return runBlocking {
+            ImportApplication.main(this)
+        }
+    }
+
     val env = System.getenv()
     val setWebhook = env["SET_WEBHOOK"]?.toBoolean() ?: false
     val monoWebhookUrl = URI(env["MONO_WEBHOOK_URL"]!!)
@@ -54,13 +61,13 @@ fun main() {
     log.debug { "\twebhookSettings: $webhookSettings" }
 
     runBlocking {
-        startKoin {
+        val koin = startKoin {
             setupKoinModules(this@runBlocking, jsonRetryRepository, settings, webhookSettings)
-        }
+        }.koin
 
         when (budgetBackend) {
-            is Lunchmoney -> Application<LunchmoneyTransaction, LunchmoneyInsertTransaction>()
-            is YNAB -> Application<YnabTransactionDetail, YnabSaveTransaction>()
+            is Lunchmoney -> Application<LunchmoneyTransaction, LunchmoneyInsertTransaction>(koin.getAll())
+            is YNAB -> Application<YnabTransactionDetail, YnabSaveTransaction>(koin.getAll())
         }.run()
     }
 }

src/main/kotlin/io/github/smaugfm/monobudget/common/BaseApplication.kt

@@ -0,0 +1,62 @@
+package io.github.smaugfm.monobudget.common
+
+import io.github.oshai.kotlinlogging.KotlinLogging
+import io.github.smaugfm.monobudget.common.exception.BudgetBackendException
+import io.github.smaugfm.monobudget.common.statement.StatementSource
+import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementEvents
+import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementItemProcessor
+import io.github.smaugfm.monobudget.common.statement.lifecycle.StatementProcessingScopeComponent
+import kotlinx.coroutines.flow.asFlow
+import kotlinx.coroutines.flow.collect
+import kotlinx.coroutines.flow.filter
+import kotlinx.coroutines.flow.flatMapMerge
+import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.flow.onEach
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+
+private val log = KotlinLogging.logger {}
+
+open class BaseApplication<TTransaction, TNewTransaction>(
+    private val statementSources: List<StatementSource>
+) : KoinComponent {
+
+    private val statementEvents by inject<StatementEvents>()
+
+    suspend fun run() {
+        beforeStart()
+
+        statementSources.forEach { it.prepare() }
+
+        afterSourcesPrepare()
+
+        log.info { "Started application" }
+
+        statementSources.asFlow()
+            .flatMapMerge { it.statements() }
+            .filter(statementEvents::onNewStatement)
+            .map(::StatementProcessingScopeComponent)
+            .onEach {
+                with(it) {
+                    try {
+                        scope.get<StatementItemProcessor<TTransaction, TNewTransaction>>()
+                            .process()
+                        statementEvents.onStatementEnd(ctx)
+                    } catch (e: BudgetBackendException) {
+                        statementEvents.onStatementRetry(ctx, e)
+                    } catch (e: Throwable) {
+                        statementEvents.onStatementError(ctx, e)
+                    } finally {
+                        scope.close()
+                    }
+                }
+            }
+            .collect()
+    }
+
+    protected open suspend fun beforeStart() {
+    }
+
+    protected open suspend fun afterSourcesPrepare() {
+    }
+}

src/main/kotlin/io/github/smaugfm/monobudget/common/account/TransferCache.kt

@@ -3,7 +3,7 @@ package io.github.smaugfm.monobudget.common.account
 import io.github.smaugfm.monobudget.common.model.financial.StatementItem
 import io.github.smaugfm.monobudget.common.util.misc.ConcurrentExpiringMap
 import kotlinx.coroutines.Deferred
-import kotlin.time.Duration.Companion.minutes
+import kotlin.time.Duration
 
-abstract class TransferCache<TTransaction> :
-    ConcurrentExpiringMap<StatementItem, Deferred<TTransaction>>(1.minutes)
+abstract class TransferCache<TTransaction>(expirationDuration: Duration) :
+    ConcurrentExpiringMap<StatementItem, Deferred<TTransaction>>(expirationDuration)

src/main/kotlin/io/github/smaugfm/monobudget/common/model/serializer/LocalDateAsISOSerializer.kt

@@ -23,6 +23,6 @@ class LocalDateAsISOSerializer : KSerializer<LocalDate> {
     }
 
     override fun deserialize(decoder: Decoder): LocalDate {
-        return decoder.decodeString().toLocalDate()
+        return LocalDate.parse(decoder.decodeString())
     }
 }

src/main/kotlin/io/github/smaugfm/monobudget/common/transaction/NewTransactionFactory.kt

@@ -24,6 +24,6 @@ abstract class NewTransactionFactory<TNewTransaction> : KoinComponent {
 
     companion object {
         @JvmStatic
-        protected fun StatementItem?.formatDescription() = (this?.description ?: "").replaceNewLines()
+        fun StatementItem?.formatDescription() = (this?.description ?: "").replaceNewLines()
     }
 }

src/main/kotlin/io/github/smaugfm/monobudget/importer/CsvMonoItem.kt

@@ -0,0 +1,26 @@
+package io.github.smaugfm.monobudget.importer
+
+import io.github.smaugfm.monobudget.common.model.financial.BankAccountId
+import io.github.smaugfm.monobudget.common.model.serializer.CurrencyAsStringSerializer
+import kotlinx.datetime.Instant
+import kotlinx.serialization.Serializable
+import java.util.Currency
+
+@Serializable
+internal data class CsvMonoItem(
+    @Serializable(MonobankInstantSerializer::class)
+    val date: Instant,
+    val description: String,
+    val mcc: Int,
+    val cardCurrencyAmount: Double,
+    val transactionCurrencyAmount: Double,
+    @Serializable(CurrencyAsStringSerializer::class)
+    val currency: Currency,
+    val exchangeRate: Double?,
+    val cardCurrencyCommissionAmount: Double?,
+    val cardCurrencyCashbackAmount: Double?,
+    val balance: Double?,
+) {
+    fun toStatementItem(accountId: BankAccountId, accountCurrency: Currency) =
+        ImportStatementItem(this, accountId, accountCurrency)
+}

src/main/kotlin/io/github/smaugfm/monobudget/importer/ImportAccountConfig.kt

@@ -0,0 +1,8 @@
+package io.github.smaugfm.monobudget.importer
+
+import io.github.smaugfm.monobudget.common.model.financial.BankAccountId
+
+data class ImportAccountConfig(
+    val accountAlias: String,
+    val transactionsFileContent: String
+)

src/main/kotlin/io/github/smaugfm/monobudget/importer/ImportApplication.kt

@@ -0,0 +1,53 @@
+package io.github.smaugfm.monobudget.importer
+
+import io.github.oshai.kotlinlogging.KotlinLogging
+import io.github.smaugfm.lunchmoney.model.LunchmoneyInsertTransaction
+import io.github.smaugfm.lunchmoney.model.LunchmoneyTransaction
+import io.github.smaugfm.monobudget.common.BaseApplication
+import io.github.smaugfm.monobudget.common.model.settings.Settings
+import io.github.smaugfm.monobudget.common.retry.InMemoryStatementRetryRepository
+import io.github.smaugfm.monobudget.lunchmoney.LunchmoneyNewTransactionFactory
+import io.github.smaugfm.monobudget.lunchmoney.LunchmoneyTransferCache
+import io.github.smaugfm.monobudget.mono.MonoWebhookSettings
+import io.github.smaugfm.monobudget.setupKoinModules
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.datetime.Clock
+import org.koin.core.context.startKoin
+import org.koin.dsl.module
+import java.net.URI
+import java.nio.file.Paths
+import kotlin.time.Duration.Companion.seconds
+
+private val log = KotlinLogging.logger {}
+
+class ImportApplication(source: ImportStatementSource) :
+    BaseApplication<LunchmoneyTransaction, LunchmoneyInsertTransaction>(listOf(source)) {
+
+    companion object {
+        suspend fun main(coroutineScope: CoroutineScope) {
+            val settings = Settings.load(
+                Paths.get(System.getenv()["SETTINGS_FILE"] ?: "settings.yml")
+            )
+            val importConfig = ImportConfig.load(
+                Paths.get(System.getenv()["IMPORT_CONFIG_FILE"] ?: "import-config.yml")
+            )
+            val noteSuffix = " monobudget-import-${Clock.System.now()}"
+            log.info { "Inserting with note suffix: \"$noteSuffix\"" }
+
+            startKoin {
+                setupKoinModules(
+                    coroutineScope,
+                    InMemoryStatementRetryRepository(),
+                    settings,
+                    MonoWebhookSettings(false, URI.create("none://none"), 0)
+                )
+                modules(module {
+                    single { LunchmoneyNewTransactionFactory(noteSuffix) }
+                    single { LunchmoneyTransferCache(Long.MAX_VALUE.seconds) }
+                })
+            }.koin
+
+            ImportApplication(ImportStatementSource(importConfig.getImports())).run()
+        }
+    }
+}

src/main/kotlin/io/github/smaugfm/monobudget/importer/ImportConfig.kt

@@ -0,0 +1,30 @@
+package io.github.smaugfm.monobudget.importer
+
+import com.charleskorn.kaml.Yaml
+import com.charleskorn.kaml.YamlConfiguration
+import io.github.oshai.kotlinlogging.KotlinLogging
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import java.io.File
+import java.nio.file.Path
+
+private val log = KotlinLogging.logger {}
+
+@Serializable
+data class ImportConfig(
+    private val imports: Map<String, String>
+) {
+    fun getImports() =
+        imports.entries.map { ImportAccountConfig(it.key, it.value) }
+
+    companion object {
+        fun load(path: Path): ImportConfig = load(File(path.toString()).readText())
+
+        private fun load(content: String): ImportConfig =
+            Yaml(configuration = YamlConfiguration(strictMode = false))
+                .decodeFromString<ImportConfig>(content)
+                .also {
+                    log.debug { "Loaded import-config: $it" }
+                }
+    }
+}

src/main/kotlin/io/github/smaugfm/monobudget/importer/ImportStatementItem.kt

@@ -0,0 +1,28 @@
+package io.github.smaugfm.monobudget.importer
+
+import io.github.smaugfm.monobudget.common.model.financial.Amount
+import io.github.smaugfm.monobudget.common.model.financial.BankAccountId
+import io.github.smaugfm.monobudget.common.model.financial.StatementItem
+import java.util.Currency
+import java.util.UUID
+
+internal data class ImportStatementItem(
+    val csv: CsvMonoItem,
+    override val accountId: BankAccountId,
+    val accountCurrency: Currency
+) : StatementItem {
+    override val id = UUID.randomUUID().toString()
+    override val time = csv.date
+    override val description = csv.description
+    override val comment = null
+    override val mcc = csv.mcc
+    override val amount = Amount.fromLunchmoneyAmount(
+        csv.cardCurrencyAmount,
+        accountCurrency
+    )
+    override val operationAmount = Amount.fromLunchmoneyAmount(
+        csv.transactionCurrencyAmount,
+        csv.currency
+    )
+    override val currency = csv.currency
+}