From d658e07583f3aed2bac0afded7c2b302e8c6411b Mon Sep 17 00:00:00 2001 From: Mitchell Syer Date: Tue, 23 Jan 2024 18:48:55 -0500 Subject: [PATCH] Implement FlareSolverr (#844) * Implement FlareSolverr * Oops --- .../kanade/tachiyomi/network/NetworkHelper.kt | 51 ++- .../network/PersistentCookieStore.kt | 14 +- .../interceptor/CloudflareInterceptor.kt | 315 +++++++++--------- .../interceptor/UserAgentInterceptor.kt | 5 +- .../tachiyomi/source/online/HttpSource.kt | 7 +- .../impl/util/source/GetCatalogueSource.kt | 4 + .../suwayomi/tachidesk/server/ServerConfig.kt | 5 + .../src/main/resources/server-reference.conf | 5 + 8 files changed, 211 insertions(+), 195 deletions(-) diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt index d48b305c6..169f990b8 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt @@ -7,30 +7,29 @@ package eu.kanade.tachiyomi.network * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -// import android.content.Context -// import eu.kanade.tachiyomi.BuildConfig -// import eu.kanade.tachiyomi.data.preference.PreferencesHelper -// import okhttp3.HttpUrl.Companion.toHttpUrl -// import okhttp3.dnsoverhttps.DnsOverHttps -// import okhttp3.logging.HttpLoggingInterceptor -// import uy.kohesive.injekt.injectLazy import android.content.Context import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor import eu.kanade.tachiyomi.network.interceptor.IgnoreGzipInterceptor import eu.kanade.tachiyomi.network.interceptor.UncaughtExceptionInterceptor import eu.kanade.tachiyomi.network.interceptor.UserAgentInterceptor +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach import mu.KotlinLogging import okhttp3.Cache import okhttp3.OkHttpClient import okhttp3.brotli.BrotliInterceptor import okhttp3.logging.HttpLoggingInterceptor +import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource import java.io.File import java.net.CookieHandler import java.net.CookieManager import java.net.CookiePolicy import java.util.concurrent.TimeUnit -@Suppress("UNUSED_PARAMETER") class NetworkHelper(context: Context) { // private val preferences: PreferencesHelper by injectLazy() @@ -48,6 +47,26 @@ class NetworkHelper(context: Context) { } // Tachidesk <-- + private val userAgent = + MutableStateFlow( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + ) + + fun defaultUserAgentProvider(): String { + return userAgent.value + } + + init { + @OptIn(DelicateCoroutinesApi::class) + userAgent + .drop(1) + .onEach { + GetCatalogueSource.unregisterAllCatalogueSources() // need to reset the headers + } + .launchIn(GlobalScope) + } + private val baseClientBuilder: OkHttpClient.Builder get() { val builder = @@ -63,7 +82,7 @@ class NetworkHelper(context: Context) { ), ) .addInterceptor(UncaughtExceptionInterceptor()) - .addInterceptor(UserAgentInterceptor()) + .addInterceptor(UserAgentInterceptor(::defaultUserAgentProvider)) .addNetworkInterceptor(IgnoreGzipInterceptor()) .addNetworkInterceptor(BrotliInterceptor) @@ -78,14 +97,14 @@ class NetworkHelper(context: Context) { } }, ).apply { - level = HttpLoggingInterceptor.Level.BASIC + level = HttpLoggingInterceptor.Level.HEADERS } builder.addNetworkInterceptor(httpLoggingInterceptor) // } - // builder.addInterceptor( - // CloudflareInterceptor(context, cookieJar, ::defaultUserAgentProvider), - // ) + builder.addInterceptor( + CloudflareInterceptor(setUserAgent = { userAgent.value = it }), + ) // when (preferences.dohProvider().get()) { // PREF_DOH_CLOUDFLARE -> builder.dohCloudflare() @@ -108,9 +127,5 @@ class NetworkHelper(context: Context) { // val client by lazy { baseClientBuilder.cache(Cache(cacheDir, cacheSize)).build() } val client by lazy { baseClientBuilder.build() } - val cloudflareClient by lazy { - client.newBuilder() - .addInterceptor(CloudflareInterceptor()) - .build() - } + val cloudflareClient by lazy { client } } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt index 9ed4d93cf..cbe82b9d2 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt @@ -21,16 +21,18 @@ class PersistentCookieStore(context: Context) : CookieStore { private val lock = ReentrantLock() init { - for ((key, value) in prefs.all) { - @Suppress("UNCHECKED_CAST") - val cookies = value as? Set - if (cookies != null) { + val domains = + prefs.all.keys.map { it.substringBeforeLast(".") } + .toSet() + domains.forEach { domain -> + val cookies = prefs.getStringSet(domain, emptySet()) + if (!cookies.isNullOrEmpty()) { try { - val url = "http://$key".toHttpUrlOrNull() ?: continue + val url = "http://$domain".toHttpUrlOrNull() ?: return@forEach val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) } .filter { !it.hasExpired() } - cookieMap.put(key, nonExpiredCookies) + cookieMap[domain] = nonExpiredCookies } catch (e: Exception) { // Ignore } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt index 2a7fe91f8..264cbccaa 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt @@ -1,39 +1,62 @@ package eu.kanade.tachiyomi.network.interceptor import eu.kanade.tachiyomi.network.NetworkHelper +import eu.kanade.tachiyomi.network.POST +import eu.kanade.tachiyomi.network.awaitSuccess +import eu.kanade.tachiyomi.network.parseAs +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json import mu.KotlinLogging +import okhttp3.Cookie +import okhttp3.HttpUrl import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response +import suwayomi.tachidesk.server.serverConfig import uy.kohesive.injekt.injectLazy import java.io.IOException +import kotlin.time.Duration.Companion.seconds -class CloudflareInterceptor : Interceptor { +class CloudflareInterceptor( + private val setUserAgent: (String) -> Unit, +) : Interceptor { private val logger = KotlinLogging.logger {} private val network: NetworkHelper by injectLazy() - @Suppress("UNUSED_VARIABLE", "UNREACHABLE_CODE") override fun intercept(chain: Interceptor.Chain): Response { val originalRequest = chain.request() logger.trace { "CloudflareInterceptor is being used." } - val originalResponse = chain.proceed(chain.request()) + val originalResponse = chain.proceed(originalRequest) // Check if Cloudflare anti-bot is on if (!(originalResponse.code in ERROR_CODES && originalResponse.header("Server") in SERVER_CHECK)) { return originalResponse } - throw IOException("Cloudflare bypass currently disabled ") + if (!serverConfig.flareSolverrEnabled.value) { + throw IOException("Cloudflare bypass currently disabled") + } logger.debug { "Cloudflare anti-bot is on, CloudflareInterceptor is kicking in..." } return try { originalResponse.close() - network.cookieStore.remove(originalRequest.url.toUri()) + // network.cookieStore.remove(originalRequest.url.toUri()) - val request = originalRequest // resolveWithWebView(originalRequest) + val request = + runBlocking { + CFClearance.resolveWithFlareSolverr(setUserAgent, originalRequest) + } chain.proceed(request) } catch (e: Exception) { @@ -57,172 +80,140 @@ class CloudflareInterceptor : Interceptor { object CFClearance { private val logger = KotlinLogging.logger {} private val network: NetworkHelper by injectLazy() - - /*init { - // Fix the default DriverJar issue by providing our own implementation - // ref: https://github.com/microsoft/playwright-java/issues/1138 - System.setProperty("playwright.driver.impl", "suwayomi.tachidesk.server.util.DriverJar") - } - - fun resolveWithWebView(originalRequest: Request): Request { - val url = originalRequest.url.toString() - - logger.debug { "resolveWithWebView($url)" } - - val cookies = - Playwright.create().use { playwright -> - playwright.chromium().launch( - LaunchOptions() - .setHeadless(false) - .apply { - if (serverConfig.socksProxyEnabled.value) { - setProxy("socks5://${serverConfig.socksProxyHost.value}:${serverConfig.socksProxyPort.value}") - } - }, - ).use { browser -> - val userAgent = originalRequest.header("User-Agent") - if (userAgent != null) { - browser.newContext(Browser.NewContextOptions().setUserAgent(userAgent)).use { browserContext -> - browserContext.newPage().use { getCookies(it, url) } - } - } else { - browser.newPage().use { getCookies(it, url) } - } + private val json: Json by injectLazy() + private val jsonMediaType = "application/json".toMediaType() + private val mutex = Mutex() + + @Serializable + data class FlareSolverCookie( + val name: String, + val value: String, + ) + + @Serializable + data class FlareSolverRequest( + val cmd: String, + val url: String, + val maxTimeout: Int? = null, + val session: List? = null, + @SerialName("session_ttl_minutes") + val sessionTtlMinutes: Int? = null, + val cookies: List? = null, + val returnOnlyCookies: Boolean? = null, + val proxy: String? = null, + val postData: String? = null, // only used with cmd 'request.post' + ) + + @Serializable + data class FlareSolverSolutionCookie( + val name: String, + val value: String, + val domain: String, + val path: String, + val expires: Double? = null, + val size: Int? = null, + val httpOnly: Boolean, + val secure: Boolean, + val session: Boolean? = null, + val sameSite: String, + ) + + @Serializable + data class FlareSolverSolution( + val url: String, + val status: Int, + val headers: Map? = null, + val response: String? = null, + val cookies: List, + val userAgent: String, + ) + + @Serializable + data class FlareSolverResponse( + val solution: FlareSolverSolution, + val status: String, + val message: String, + val startTimestamp: Long, + val endTimestamp: Long, + val version: String, + ) + + suspend fun resolveWithFlareSolverr( + setUserAgent: (String) -> Unit, + originalRequest: Request, + ): Request { + val flareSolverResponse = + with(json) { + mutex.withLock { + network.client.newCall( + POST( + url = serverConfig.flareSolverrUrl.value.removeSuffix("/") + "/v1", + body = + Json.encodeToString( + FlareSolverRequest( + "request.get", + originalRequest.url.toString(), + cookies = + network.cookieStore.get(originalRequest.url).map { + FlareSolverCookie(it.name, it.value) + }, + returnOnlyCookies = true, + maxTimeout = + serverConfig.flareSolverrTimeout.value + .seconds + .inWholeMilliseconds + .toInt(), + ), + ).toRequestBody(jsonMediaType), + ), + ).awaitSuccess().parseAs() } } - // Copy cookies to cookie store - cookies.groupBy { it.domain }.forEach { (domain, cookies) -> - network.cookieStore.addAll( - url = - HttpUrl.Builder() - .scheme("http") - .host(domain) - .build(), - cookies = cookies, - ) - } - // Merge new and existing cookies for this request - // Find the cookies that we need to merge into this request - val convertedForThisRequest = - cookies.filter { - it.matches(originalRequest.url) - } - // Extract cookies from current request - val existingCookies = - Cookie.parseAll( - originalRequest.url, - originalRequest.headers, - ) - // Filter out existing values of cookies that we are about to merge in - val filteredExisting = - existingCookies.filter { existing -> - convertedForThisRequest.none { converted -> converted.name == existing.name } - } - logger.trace { "Existing cookies" } - logger.trace { existingCookies.joinToString("; ") } - val newCookies = filteredExisting + convertedForThisRequest - logger.trace { "New cookies" } - logger.trace { newCookies.joinToString("; ") } - return originalRequest.newBuilder() - .header("Cookie", newCookies.joinToString("; ") { "${it.name}=${it.value}" }) - .build() - }*/ - - fun getWebViewUserAgent(): String { - return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" - /*return try { - throw PlaywrightException("playwrite is diabled for v0.6.7") - - Playwright.create().use { playwright -> - playwright.chromium().launch( - LaunchOptions() - .setHeadless(true), - ).use { browser -> - browser.newPage().use { page -> - val userAgent = page.evaluate("() => {return navigator.userAgent}") as String - logger.debug { "WebView User-Agent is $userAgent" } - return userAgent + if (flareSolverResponse.solution.status in 200..299) { + setUserAgent(flareSolverResponse.solution.userAgent) + val cookies = + flareSolverResponse.solution.cookies + .map { cookie -> + Cookie.Builder() + .name(cookie.name) + .value(cookie.value) + .domain(cookie.domain) + .path(cookie.path) + .expiresAt(cookie.expires?.takeUnless { it < 0.0 }?.toLong() ?: Long.MAX_VALUE) + .also { + if (cookie.httpOnly) it.httpOnly() + if (cookie.secure) it.secure() + } + .build() + } + .groupBy { it.domain } + .flatMap { (domain, cookies) -> + network.cookieStore.addAll( + HttpUrl.Builder() + .scheme("http") + .host(domain.removePrefix(".")) + .build(), + cookies, + ) + + cookies } + logger.trace { "New cookies\n${cookies.joinToString("; ")}" } + val finalCookies = + network.cookieStore.get(originalRequest.url).joinToString("; ", postfix = "; ") { + "${it.name}=${it.value}" } - } - } catch (e: PlaywrightException) { - // Playwright might fail on headless environments like docker - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36" - }*/ - } - - /*private fun getCookies( - page: Page, - url: String, - ): List { - applyStealthInitScripts(page) - page.navigate(url) - val challengeResolved = waitForChallengeResolve(page) - - return if (challengeResolved) { - val cookies = page.context().cookies() - - logger.debug { - val userAgent = page.evaluate("() => {return navigator.userAgent}") - "Playwright User-Agent is $userAgent" - } - - // Convert PlayWright cookies to OkHttp cookies - cookies.map { - Cookie.Builder() - .domain(it.domain.removePrefix(".")) - .expiresAt(it.expires?.times(1000)?.toLong() ?: Long.MAX_VALUE) - .name(it.name) - .path(it.path) - .value(it.value).apply { - if (it.httpOnly) httpOnly() - if (it.secure) secure() - }.build() - } + logger.trace { "Final cookies\n$finalCookies" } + return originalRequest.newBuilder() + .header("Cookie", finalCookies) + .header("User-Agent", flareSolverResponse.solution.userAgent) + .build() } else { logger.debug { "Cloudflare challenge failed to resolve" } throw CloudflareBypassException() } } - // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L18 - private val stealthInitScripts by lazy { - arrayOf( - ServerConfig::class.java.getResource("/cloudflare-js/canvas.fingerprinting.js")!!.readText(), - ServerConfig::class.java.getResource("/cloudflare-js/chrome.global.js")!!.readText(), - ServerConfig::class.java.getResource("/cloudflare-js/emulate.touch.js")!!.readText(), - ServerConfig::class.java.getResource("/cloudflare-js/navigator.permissions.js")!!.readText(), - ServerConfig::class.java.getResource("/cloudflare-js/navigator.webdriver.js")!!.readText(), - ServerConfig::class.java.getResource("/cloudflare-js/chrome.runtime.js")!!.readText(), - ServerConfig::class.java.getResource("/cloudflare-js/chrome.plugin.js")!!.readText(), - ) - } - - // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/stealth.py#L76 - private fun applyStealthInitScripts(page: Page) { - for (script in stealthInitScripts) { - page.addInitScript(script) - } - } - - // ref: https://github.com/vvanglro/cf-clearance/blob/44124a8f06d8d0ecf2bf558a027082ff88dab435/cf_clearance/retry.py#L21 - private fun waitForChallengeResolve(page: Page): Boolean { - // sometimes the user has to solve the captcha challenge manually, potentially wait a long time - val timeoutSeconds = 120 - repeat(timeoutSeconds) { - page.waitForTimeout(1.seconds.toDouble(DurationUnit.MILLISECONDS)) - val success = - try { - page.querySelector("#challenge-form") == null - } catch (e: Exception) { - logger.debug(e) { "query Error" } - false - } - if (success) return true - } - return false - }*/ - private class CloudflareBypassException : Exception() } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt index 5488c8775..946832378 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt @@ -1,10 +1,9 @@ package eu.kanade.tachiyomi.network.interceptor -import eu.kanade.tachiyomi.source.online.HttpSource import okhttp3.Interceptor import okhttp3.Response -class UserAgentInterceptor : Interceptor { +class UserAgentInterceptor(private val userAgentProvider: () -> String) : Interceptor { override fun intercept(chain: Interceptor.Chain): Response { val originalRequest = chain.request() @@ -13,7 +12,7 @@ class UserAgentInterceptor : Interceptor { originalRequest .newBuilder() .removeHeader("User-Agent") - .addHeader("User-Agent", HttpSource.DEFAULT_USER_AGENT) + .addHeader("User-Agent", userAgentProvider()) .build() chain.proceed(newRequest) } else { diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt index 55252757b..88b9dc70a 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt @@ -4,7 +4,6 @@ import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.NetworkHelper import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.network.awaitSuccess -import eu.kanade.tachiyomi.network.interceptor.CFClearance.getWebViewUserAgent import eu.kanade.tachiyomi.network.newCachelessCallWithProgress import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.model.FilterList @@ -107,7 +106,7 @@ abstract class HttpSource : CatalogueSource { */ protected open fun headersBuilder() = Headers.Builder().apply { - add("User-Agent", DEFAULT_USER_AGENT) + add("User-Agent", network.defaultUserAgentProvider()) } /** @@ -480,10 +479,6 @@ abstract class HttpSource : CatalogueSource { * Returns the list of filters for the source. */ override fun getFilterList() = FilterList() - - companion object { - val DEFAULT_USER_AGENT by lazy { getWebViewUserAgent() } - } } class LicensedMangaChaptersException : Exception("Licensed - No chapters to show") diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt index 520e71183..f3843e048 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/source/GetCatalogueSource.kt @@ -81,4 +81,8 @@ object GetCatalogueSource { fun unregisterCatalogueSource(sourceId: Long) { sourceCache.remove(sourceId) } + + fun unregisterAllCatalogueSources() { + sourceCache.clear() + } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index 227721fad..ac63ebc87 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -131,6 +131,11 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF // local source val localSourcePath: MutableStateFlow by OverrideConfigValue(StringConfigAdapter) + // cloudflare bypass + val flareSolverrEnabled: MutableStateFlow by OverrideConfigValue(BooleanConfigAdapter) + val flareSolverrUrl: MutableStateFlow by OverrideConfigValue(StringConfigAdapter) + val flareSolverrTimeout: MutableStateFlow by OverrideConfigValue(IntConfigAdapter) + @OptIn(ExperimentalCoroutinesApi::class) fun subscribeTo( flow: Flow, diff --git a/server/src/main/resources/server-reference.conf b/server/src/main/resources/server-reference.conf index 70f38a3db..0588defe1 100644 --- a/server/src/main/resources/server-reference.conf +++ b/server/src/main/resources/server-reference.conf @@ -56,3 +56,8 @@ server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - d # local source server.localSourcePath = "" + +# Cloudflare bypass +server.flareSolverrEnabled = false +server.flareSolverrUrl = "http://localhost:8191" +server.flareSolverrTimeout = 60 # time in seconds \ No newline at end of file