Feature/remove download ahead logic (#867)

* Remove download ahead logic

Unnecessary on server side, should just be done by the client

* Rename "autoDownloadAheadLimit" to "autoDownloadNewChaptersLimit"

* Deprecate the old field

* Update Stable WebUI

* Update Stable WebUI

---------

Co-authored-by: Syer10 <[email protected]>

Author: schroda

buildSrc/src/main/kotlin/Constants.kt

@@ -12,7 +12,7 @@ const val MainClass = "suwayomi.tachidesk.MainKt"
 // should be bumped with each stable release
 val tachideskVersion = System.getenv("ProductVersion") ?: "v0.7.0"
 
-val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r1335"
+val webUIRevisionTag = System.getenv("WebUIRevision") ?: "r1397"
 
 // counts commits on the current checked out branch
 val getTachideskRevision = {

server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/DownloadMutation.kt

@@ -7,7 +7,6 @@ import org.jetbrains.exposed.sql.transactions.transaction
 import suwayomi.tachidesk.graphql.types.ChapterType
 import suwayomi.tachidesk.graphql.types.DownloadStatus
 import suwayomi.tachidesk.manga.impl.Chapter
-import suwayomi.tachidesk.manga.impl.Manga
 import suwayomi.tachidesk.manga.impl.download.DownloadManager
 import suwayomi.tachidesk.manga.impl.download.model.Status
 import suwayomi.tachidesk.manga.model.table.ChapterTable
@@ -269,20 +268,4 @@ class DownloadMutation {
             )
         }
     }
-
-    data class DownloadAheadInput(
-        val clientMutationId: String? = null,
-        val mangaIds: List<Int> = emptyList(),
-        val latestReadChapterIds: List<Int>? = null,
-    )
-
-    data class DownloadAheadPayload(val clientMutationId: String?)
-
-    fun downloadAhead(input: DownloadAheadInput): DownloadAheadPayload {
-        val (clientMutationId, mangaIds, latestReadChapterIds) = input
-
-        Manga.downloadAhead(mangaIds, latestReadChapterIds ?: emptyList())
-
-        return DownloadAheadPayload(clientMutationId)
-    }
 }

server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt

@@ -53,7 +53,8 @@ class SettingsMutation {
         updateSetting(settings.downloadsPath, serverConfig.downloadsPath)
         updateSetting(settings.autoDownloadNewChapters, serverConfig.autoDownloadNewChapters)
         updateSetting(settings.excludeEntryWithUnreadChapters, serverConfig.excludeEntryWithUnreadChapters)
-        updateSetting(settings.autoDownloadAheadLimit, serverConfig.autoDownloadAheadLimit)
+        updateSetting(settings.autoDownloadAheadLimit, serverConfig.autoDownloadNewChaptersLimit) // deprecated
+        updateSetting(settings.autoDownloadNewChaptersLimit, serverConfig.autoDownloadNewChaptersLimit)
 
         // extension
         updateSetting(settings.extensionRepos, serverConfig.extensionRepos)

server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt

@@ -7,6 +7,7 @@
 
 package suwayomi.tachidesk.graphql.types
 
+import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated
 import suwayomi.tachidesk.graphql.server.primitives.Node
 import suwayomi.tachidesk.server.ServerConfig
 import suwayomi.tachidesk.server.serverConfig
@@ -38,7 +39,13 @@ interface Settings : Node {
     val downloadsPath: String?
     val autoDownloadNewChapters: Boolean?
     val excludeEntryWithUnreadChapters: Boolean?
+
+    @GraphQLDeprecated(
+        "Replaced with autoDownloadNewChaptersLimit",
+        replaceWith = ReplaceWith("autoDownloadNewChaptersLimit"),
+    )
     val autoDownloadAheadLimit: Int?
+    val autoDownloadNewChaptersLimit: Int?
 
     // extension
     val extensionRepos: List<String>?
@@ -99,7 +106,12 @@ data class PartialSettingsType(
     override val downloadsPath: String?,
     override val autoDownloadNewChapters: Boolean?,
     override val excludeEntryWithUnreadChapters: Boolean?,
+    @GraphQLDeprecated(
+        "Replaced with autoDownloadNewChaptersLimit",
+        replaceWith = ReplaceWith("autoDownloadNewChaptersLimit"),
+    )
     override val autoDownloadAheadLimit: Int?,
+    override val autoDownloadNewChaptersLimit: Int?,
     // extension
     override val extensionRepos: List<String>?,
     // requests
@@ -152,7 +164,12 @@ class SettingsType(
     override val downloadsPath: String,
     override val autoDownloadNewChapters: Boolean,
     override val excludeEntryWithUnreadChapters: Boolean,
+    @GraphQLDeprecated(
+        "Replaced with autoDownloadNewChaptersLimit",
+        replaceWith = ReplaceWith("autoDownloadNewChaptersLimit"),
+    )
     override val autoDownloadAheadLimit: Int,
+    override val autoDownloadNewChaptersLimit: Int,
     // extension
     override val extensionRepos: List<String>,
     // requests
@@ -204,7 +221,8 @@ class SettingsType(
         config.downloadsPath.value,
         config.autoDownloadNewChapters.value,
         config.excludeEntryWithUnreadChapters.value,
-        config.autoDownloadAheadLimit.value,
+        config.autoDownloadNewChaptersLimit.value, // deprecated
+        config.autoDownloadNewChaptersLimit.value,
         // extension
         config.extensionRepos.value,
         // requests

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -310,7 +310,7 @@ object Chapter {
                     "mangaId= $mangaId, " +
                     "prevNumberOfChapters= $prevNumberOfChapters, " +
                     "updatedChapterList= ${updatedChapterList.size}, " +
-                    "downloadAheadLimit= ${serverConfig.autoDownloadAheadLimit.value}" +
+                    "autoDownloadNewChaptersLimit= ${serverConfig.autoDownloadNewChaptersLimit.value}" +
                     ")",
             )
 
@@ -389,8 +389,8 @@ object Chapter {
         }
 
         val firstChapterToDownloadIndex =
-            if (serverConfig.autoDownloadAheadLimit.value > 0) {
-                (numberOfNewChapters - serverConfig.autoDownloadAheadLimit.value).coerceAtLeast(0)
+            if (serverConfig.autoDownloadNewChaptersLimit.value > 0) {
+                (numberOfNewChapters - serverConfig.autoDownloadNewChaptersLimit.value).coerceAtLeast(0)
             } else {
                 0
             }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -27,8 +27,6 @@ import org.kodein.di.conf.global
 import org.kodein.di.instance
 import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl
 import suwayomi.tachidesk.manga.impl.Source.getSource
-import suwayomi.tachidesk.manga.impl.download.DownloadManager
-import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput
 import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.MissingThumbnailException
 import suwayomi.tachidesk.manga.impl.track.Track
 import suwayomi.tachidesk.manga.impl.util.network.await
@@ -47,15 +45,11 @@ import suwayomi.tachidesk.manga.model.table.MangaStatus
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
 import suwayomi.tachidesk.server.ApplicationDirs
-import suwayomi.tachidesk.server.serverConfig
 import uy.kohesive.injekt.injectLazy
 import java.io.File
 import java.io.IOException
 import java.io.InputStream
 import java.time.Instant
-import java.util.Timer
-import java.util.TimerTask
-import java.util.concurrent.ConcurrentHashMap
 
 private val logger = KotlinLogging.logger { }
 
@@ -339,104 +333,4 @@ object Manga {
         clearCachedImage(applicationDirs.tempThumbnailCacheRoot, fileName)
         clearCachedImage(applicationDirs.thumbnailDownloadsRoot, fileName)
     }
-
-    private val downloadAheadQueue = ConcurrentHashMap<String, ConcurrentHashMap.KeySetView<Int, Boolean>>()
-    private var downloadAheadTimer: Timer? = null
-
-    private const val MANGAS_KEY = "mangaIds"
-    private const val CHAPTERS_KEY = "chapterIds"
-
-    fun downloadAhead(
-        mangaIds: List<Int>,
-        latestReadChapterIds: List<Int> = emptyList(),
-    ) {
-        if (serverConfig.autoDownloadAheadLimit.value == 0) {
-            return
-        }
-
-        val updateDownloadAheadQueue = { key: String, ids: List<Int> ->
-            val idSet = downloadAheadQueue[key] ?: ConcurrentHashMap.newKeySet()
-            idSet.addAll(ids)
-            downloadAheadQueue[key] = idSet
-        }
-
-        updateDownloadAheadQueue(MANGAS_KEY, mangaIds)
-        updateDownloadAheadQueue(CHAPTERS_KEY, latestReadChapterIds)
-
-        // handle cases where this function gets called multiple times in quick succession.
-        // this could happen in case e.g. multiple chapters get marked as read without batching the operation
-        downloadAheadTimer?.cancel()
-        downloadAheadTimer =
-            Timer().apply {
-                schedule(
-                    object : TimerTask() {
-                        override fun run() {
-                            downloadAheadChapters(
-                                downloadAheadQueue[MANGAS_KEY]?.toList().orEmpty(),
-                                downloadAheadQueue[CHAPTERS_KEY]?.toList().orEmpty(),
-                            )
-                            downloadAheadQueue.clear()
-                        }
-                    },
-                    5000,
-                )
-            }
-    }
-
-    /**
-     * Downloads the latest unread and not downloaded chapters for each passed manga id.
-     *
-     * To pass a specific chapter as the latest read chapter for a manga, it can be provided in the "latestReadChapterIds" list.
-     * This makes it possible to handle cases, where the actual latest read chapter isn't marked as read yet.
-     * E.g. the client marks a chapter as read and at the same time sends the "downloadAhead" mutation.
-     * In this case, the latest read chapter could potentially be the one, that just got send to get marked as read by the client.
-     * Without providing it in "latestReadChapterIds" it could be incorrectly included in the chapters, that will get downloaded.
-     *
-     * The latest read chapter will be considered the starting point.
-     * E.g.:
-     * - 20 chapters
-     * - chapter 15 marked as read
-     * - 16 - 20 marked as unread
-     * - 10 - 14 marked as unread
-     *
-     * will download the unread chapters starting from chapter 15
-     */
-    private fun downloadAheadChapters(
-        mangaIds: List<Int>,
-        latestReadChapterIds: List<Int>,
-    ) {
-        val mangaToLatestReadChapterIndex =
-            transaction {
-                ChapterTable.select { (ChapterTable.manga inList mangaIds) and (ChapterTable.isRead eq true) }
-                    .orderBy(ChapterTable.sourceOrder to SortOrder.DESC).groupBy { it[ChapterTable.manga].value }
-            }.mapValues { (_, chapters) -> chapters.firstOrNull()?.let { it[ChapterTable.sourceOrder] } ?: 0 }
-
-        val mangaToUnreadChaptersMap =
-            transaction {
-                ChapterTable.select { (ChapterTable.manga inList mangaIds) and (ChapterTable.isRead eq false) }
-                    .orderBy(ChapterTable.sourceOrder to SortOrder.DESC)
-                    .groupBy { it[ChapterTable.manga].value }
-            }
-
-        val chapterIdsToDownload =
-            mangaToUnreadChaptersMap.map { (mangaId, unreadChapters) ->
-                val latestReadChapterIndex = mangaToLatestReadChapterIndex[mangaId] ?: 0
-                val lastChapterToDownloadIndex =
-                    unreadChapters.indexOfLast {
-                        it[ChapterTable.sourceOrder] > latestReadChapterIndex &&
-                            it[ChapterTable.id].value !in latestReadChapterIds
-                    }
-                val unreadChaptersToConsider = unreadChapters.subList(0, lastChapterToDownloadIndex + 1)
-                val firstChapterToDownloadIndex =
-                    (unreadChaptersToConsider.size - serverConfig.autoDownloadAheadLimit.value).coerceAtLeast(0)
-                unreadChaptersToConsider.subList(firstChapterToDownloadIndex, lastChapterToDownloadIndex + 1)
-                    .filter { !it[ChapterTable.isDownloaded] }
-                    .map { it[ChapterTable.id].value }
-            }.flatten()
-
-        logger.info { "downloadAheadChapters: download chapters [${chapterIdsToDownload.joinToString(", ")}]" }
-
-        DownloadManager.dequeue(mangaIds, chapterIdsToDownload)
-        DownloadManager.enqueue(EnqueueInput(chapterIdsToDownload))
-    }
 }

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -97,7 +97,7 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF
     val downloadsPath: MutableStateFlow<String> by OverrideConfigValue(StringConfigAdapter)
     val autoDownloadNewChapters: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
     val excludeEntryWithUnreadChapters: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
-    val autoDownloadAheadLimit: MutableStateFlow<Int> by OverrideConfigValue(IntConfigAdapter)
+    val autoDownloadNewChaptersLimit: MutableStateFlow<Int> by OverrideConfigValue(IntConfigAdapter)
 
     // extensions
     val extensionRepos: MutableStateFlow<List<String>> by OverrideConfigValues(StringConfigAdapter)

server/src/main/resources/server-reference.conf

@@ -21,7 +21,7 @@ server.downloadAsCbz = false
 server.downloadsPath = ""
 server.autoDownloadNewChapters = false # if new chapters that have been retrieved should get automatically downloaded
 server.excludeEntryWithUnreadChapters = true # ignore automatic chapter downloads of entries with unread chapters
-server.autoDownloadAheadLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update
+server.autoDownloadNewChaptersLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update
 
 # extension repos
 server.extensionRepos = [

server/src/test/resources/server-reference.conf

@@ -11,7 +11,7 @@ server.socksProxyPort = ""
 server.downloadAsCbz = false
 server.autoDownloadNewChapters = false
 server.excludeEntryWithUnreadChapters = true
-server.autoDownloadAheadLimit = 0
+server.autoDownloadNewChaptersLimit = 0
 
 # requests
 server.maxSourcesInParallel = 10