From e7c9218c05d3965b33baef78d9df5a8d3b09c484 Mon Sep 17 00:00:00 2001 From: Osip Fatkullin Date: Fri, 10 Jan 2025 12:52:12 +0100 Subject: [PATCH] KTOR-8043 Retry server tests one time by default --- .../ktor-server-test-base/build.gradle.kts | 5 +- .../src/io/ktor/server/test/base/BaseTest.kt | 28 ++++------ .../test/base/BaseTest.jsAndWasmShared.kt | 52 ------------------- .../io/ktor/server/test/base/BaseTestJvm.kt | 22 +++++--- .../ktor/server/test/base/BaseTest.nonJvm.kt} | 26 ++++++---- .../common/src/io/ktor/test/TestResult.kt | 21 +++++++- .../test/io/ktor/test/RunTestWithDataTest.kt | 10 ++-- .../ktor/test/TestResult.jsAndWasmShared.kt | 5 +- .../src/io/ktor/test/junit/ErrorCollector.kt | 6 ++- .../io/ktor/test/TestResult.jvmAndPosix.kt | 4 +- 10 files changed, 79 insertions(+), 100 deletions(-) delete mode 100644 ktor-server/ktor-server-test-base/jsAndWasmShared/src/io/ktor/server/test/base/BaseTest.jsAndWasmShared.kt rename ktor-server/ktor-server-test-base/{posix/src/io/ktor/server/test/base/BaseTestNix.kt => nonJvm/src/io/ktor/server/test/base/BaseTest.nonJvm.kt} (73%) diff --git a/ktor-server/ktor-server-test-base/build.gradle.kts b/ktor-server/ktor-server-test-base/build.gradle.kts index 9f9c3a183f..33f6b5f155 100644 --- a/ktor-server/ktor-server-test-base/build.gradle.kts +++ b/ktor-server/ktor-server-test-base/build.gradle.kts @@ -1,5 +1,5 @@ /* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ description = "" @@ -10,7 +10,7 @@ kotlin.sourceSets { commonMain { dependencies { api(project(":ktor-server:ktor-server-test-host")) - api(libs.kotlin.test) + api(project(":ktor-shared:ktor-test-base")) } } @@ -21,7 +21,6 @@ kotlin.sourceSets { api(project(":ktor-client:ktor-client-apache")) api(project(":ktor-network:ktor-network-tls:ktor-network-tls-certificates")) api(project(":ktor-server:ktor-server-plugins:ktor-server-call-logging")) - api(project(":ktor-shared:ktor-test-base")) if (jetty_alpn_boot_version != null) { api(libs.jetty.alpn.boot) diff --git a/ktor-server/ktor-server-test-base/common/src/io/ktor/server/test/base/BaseTest.kt b/ktor-server/ktor-server-test-base/common/src/io/ktor/server/test/base/BaseTest.kt index 5147146ca7..6e3a377b1a 100644 --- a/ktor-server/ktor-server-test-base/common/src/io/ktor/server/test/base/BaseTest.kt +++ b/ktor-server/ktor-server-test-base/common/src/io/ktor/server/test/base/BaseTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ package io.ktor.server.test.base @@ -16,21 +16,15 @@ expect abstract class BaseTest() { open fun afterTest() fun collectUnhandledException(error: Throwable) // TODO: better name? - fun runTest(timeout: Duration = 60.seconds, block: suspend CoroutineScope.() -> Unit): TestResult + fun runTest( + timeout: Duration = 60.seconds, + retries: Int = DEFAULT_RETRIES, + block: suspend CoroutineScope.() -> Unit + ): TestResult } -fun BaseTest.runTest( - retry: Int, - timeout: Duration = this.timeout, - block: suspend CoroutineScope.() -> Unit -): TestResult { - lateinit var lastCause: Throwable - repeat(retry) { - try { - return runTest(timeout, block) - } catch (cause: Throwable) { - lastCause = cause - } - } - throw lastCause -} +/** + * Defaults to `1` on all platforms except for JVM. + * On JVM retries are disabled as we use test-retry Gradle plugin instead. + */ +internal expect val DEFAULT_RETRIES: Int diff --git a/ktor-server/ktor-server-test-base/jsAndWasmShared/src/io/ktor/server/test/base/BaseTest.jsAndWasmShared.kt b/ktor-server/ktor-server-test-base/jsAndWasmShared/src/io/ktor/server/test/base/BaseTest.jsAndWasmShared.kt deleted file mode 100644 index e2555e891d..0000000000 --- a/ktor-server/ktor-server-test-base/jsAndWasmShared/src/io/ktor/server/test/base/BaseTest.jsAndWasmShared.kt +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. - */ - -package io.ktor.server.test.base - -import io.ktor.test.dispatcher.* -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.test.TestResult -import kotlin.time.Duration -import kotlin.time.Duration.Companion.seconds - -actual abstract class BaseTest actual constructor() { - actual open val timeout: Duration = 10.seconds - - private val errors = mutableListOf() - - actual fun collectUnhandledException(error: Throwable) { - errors.add(error) - } - - actual open fun beforeTest() { - } - - actual open fun afterTest() { - if (errors.isEmpty()) return - - val error = UnhandledErrorsException( - "There were ${errors.size} unhandled errors during running test (suppressed)" - ) - - errors.forEach { - error.addSuppressed(it) - } - error.printStackTrace() - throw error // suppressed exceptions print wrong in idea - } - - actual fun runTest( - timeout: Duration, - block: suspend CoroutineScope.() -> Unit - ): TestResult = runTestWithRealTime(timeout = timeout) { - beforeTest() - try { - block() - } finally { - afterTest() - } - } -} - -private class UnhandledErrorsException(override val message: String) : Exception() diff --git a/ktor-server/ktor-server-test-base/jvm/src/io/ktor/server/test/base/BaseTestJvm.kt b/ktor-server/ktor-server-test-base/jvm/src/io/ktor/server/test/base/BaseTestJvm.kt index b12de8382f..39d0d7795d 100644 --- a/ktor-server/ktor-server-test-base/jvm/src/io/ktor/server/test/base/BaseTestJvm.kt +++ b/ktor-server/ktor-server-test-base/jvm/src/io/ktor/server/test/base/BaseTestJvm.kt @@ -1,9 +1,10 @@ /* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ package io.ktor.server.test.base +import io.ktor.test.* import io.ktor.test.dispatcher.* import io.ktor.test.junit.* import io.ktor.test.junit.coroutines.* @@ -39,6 +40,7 @@ actual abstract class BaseTest actual constructor() { } actual open fun beforeTest() { + errorCollector.clear() } actual fun collectUnhandledException(error: Throwable) { @@ -47,13 +49,19 @@ actual abstract class BaseTest actual constructor() { actual fun runTest( timeout: Duration, + retries: Int, block: suspend CoroutineScope.() -> Unit - ): TestResult = runTestWithRealTime(CoroutineName("test-$testName"), timeout) { - beforeTest() - try { - block() - } finally { - afterTest() + ): TestResult = retryTest(retries) { + runTestWithRealTime(CoroutineName("test-$testName"), timeout) { + beforeTest() + try { + block() + } finally { + afterTest() + } } } } + +/** On JVM retries are disabled as we use test-retry Gradle plugin instead. */ +internal actual const val DEFAULT_RETRIES: Int = 0 diff --git a/ktor-server/ktor-server-test-base/posix/src/io/ktor/server/test/base/BaseTestNix.kt b/ktor-server/ktor-server-test-base/nonJvm/src/io/ktor/server/test/base/BaseTest.nonJvm.kt similarity index 73% rename from ktor-server/ktor-server-test-base/posix/src/io/ktor/server/test/base/BaseTestNix.kt rename to ktor-server/ktor-server-test-base/nonJvm/src/io/ktor/server/test/base/BaseTest.nonJvm.kt index b50f9bba0e..4a8acfc3f2 100644 --- a/ktor-server/ktor-server-test-base/posix/src/io/ktor/server/test/base/BaseTestNix.kt +++ b/ktor-server/ktor-server-test-base/nonJvm/src/io/ktor/server/test/base/BaseTest.nonJvm.kt @@ -1,9 +1,10 @@ /* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ package io.ktor.server.test.base +import io.ktor.test.* import io.ktor.test.dispatcher.* import io.ktor.utils.io.* import io.ktor.utils.io.locks.* @@ -12,15 +13,14 @@ import kotlinx.coroutines.test.TestResult import kotlin.time.Duration import kotlin.time.Duration.Companion.seconds +@OptIn(InternalAPI::class) actual abstract class BaseTest actual constructor() { actual open val timeout: Duration = 10.seconds private val errors = mutableListOf() - @OptIn(InternalAPI::class) private val errorsLock = SynchronizedObject() - @OptIn(InternalAPI::class) actual fun collectUnhandledException(error: Throwable) { synchronized(errorsLock) { errors.add(error) @@ -28,6 +28,9 @@ actual abstract class BaseTest actual constructor() { } actual open fun beforeTest() { + synchronized(errorsLock) { + errors.clear() + } } actual open fun afterTest() { @@ -46,15 +49,20 @@ actual abstract class BaseTest actual constructor() { actual fun runTest( timeout: Duration, + retries: Int, block: suspend CoroutineScope.() -> Unit - ): TestResult = runTestWithRealTime(timeout = timeout) { - beforeTest() - try { - block() - } finally { - afterTest() + ): TestResult = retryTest(retries) { + runTestWithRealTime(timeout = timeout) { + beforeTest() + try { + block() + } finally { + afterTest() + } } } } +internal actual const val DEFAULT_RETRIES: Int = 1 + private class UnhandledErrorsException(override val message: String) : Exception() diff --git a/ktor-shared/ktor-test-base/common/src/io/ktor/test/TestResult.kt b/ktor-shared/ktor-test-base/common/src/io/ktor/test/TestResult.kt index 43406392bd..51609bde12 100644 --- a/ktor-shared/ktor-test-base/common/src/io/ktor/test/TestResult.kt +++ b/ktor-shared/ktor-test-base/common/src/io/ktor/test/TestResult.kt @@ -1,5 +1,5 @@ /* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ package io.ktor.test @@ -18,4 +18,21 @@ expect inline fun TestResult.andThen(crossinline block: () -> Any): TestResult internal expect inline fun testWithRecover(noinline recover: (Throwable) -> Unit, test: () -> TestResult): TestResult internal expect inline fun runTestForEach(items: Iterable, crossinline test: (T) -> TestResult): TestResult -internal expect inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult + +/** + * Executes a test function with retry capabilities. + * + * ``` + * retryTest(retires = 2) { retry -> + * runTest { + * println("This test passes only on second retry. Current retry is $retry") + * assertEquals(2, retry) + * } + * } + * ``` + * + * @param retries The number of retries to attempt after an initial failure. Must be a non-negative integer. + * @param test A test to execute, which accepts the current retry attempt (starting at 0) as an argument. + * @return A [TestResult] representing the outcome of the test after all attempts. + */ +expect inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult diff --git a/ktor-shared/ktor-test-base/common/test/io/ktor/test/RunTestWithDataTest.kt b/ktor-shared/ktor-test-base/common/test/io/ktor/test/RunTestWithDataTest.kt index 18454020c1..1a5f4afaf9 100644 --- a/ktor-shared/ktor-test-base/common/test/io/ktor/test/RunTestWithDataTest.kt +++ b/ktor-shared/ktor-test-base/common/test/io/ktor/test/RunTestWithDataTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ package io.ktor.test @@ -101,9 +101,9 @@ class RunTestWithDataTest { fun testRetriesHaveIndependentTimeout() = runTestWithData( singleTestCase, retries = 1, - timeout = 30.milliseconds, + timeout = 50.milliseconds, test = { (_, retry) -> - realTimeDelay(20.milliseconds) + realTimeDelay(30.milliseconds) if (retry == 0) fail("Try again, please") }, ) @@ -111,8 +111,8 @@ class RunTestWithDataTest { @Test fun testDifferentItemsHaveIndependentTimeout() = runTestWithData( testCases = 1..2, - timeout = 30.milliseconds, - test = { realTimeDelay(20.milliseconds) }, + timeout = 50.milliseconds, + test = { realTimeDelay(30.milliseconds) }, ) @Test diff --git a/ktor-shared/ktor-test-base/jsAndWasmShared/src/io/ktor/test/TestResult.jsAndWasmShared.kt b/ktor-shared/ktor-test-base/jsAndWasmShared/src/io/ktor/test/TestResult.jsAndWasmShared.kt index 2526da9ffa..a272e7d239 100644 --- a/ktor-shared/ktor-test-base/jsAndWasmShared/src/io/ktor/test/TestResult.jsAndWasmShared.kt +++ b/ktor-shared/ktor-test-base/jsAndWasmShared/src/io/ktor/test/TestResult.jsAndWasmShared.kt @@ -1,5 +1,5 @@ /* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ package io.ktor.test @@ -14,7 +14,8 @@ internal actual inline fun testWithRecover( internal actual inline fun runTestForEach(items: Iterable, crossinline test: (T) -> TestResult): TestResult = items.fold(DummyTestResult) { acc, item -> acc.andThen { test(item) } } -internal actual inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult = +actual inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult = (1..retries).fold(test(0)) { acc, retry -> acc.catch { test(retry) } } +@PublishedApi internal expect fun TestResult.catch(action: (Throwable) -> Any): TestResult diff --git a/ktor-shared/ktor-test-base/jvm/src/io/ktor/test/junit/ErrorCollector.kt b/ktor-shared/ktor-test-base/jvm/src/io/ktor/test/junit/ErrorCollector.kt index 5654fd71e3..60c58d8063 100644 --- a/ktor-shared/ktor-test-base/jvm/src/io/ktor/test/junit/ErrorCollector.kt +++ b/ktor-shared/ktor-test-base/jvm/src/io/ktor/test/junit/ErrorCollector.kt @@ -1,5 +1,5 @@ /* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ package io.ktor.test.junit @@ -50,4 +50,8 @@ class ErrorCollector : TestExecutionExceptionHandler, ParameterResolver, AfterEa } } } + + fun clear() { + errors.clear() + } } diff --git a/ktor-shared/ktor-test-base/jvmAndPosix/src/io/ktor/test/TestResult.jvmAndPosix.kt b/ktor-shared/ktor-test-base/jvmAndPosix/src/io/ktor/test/TestResult.jvmAndPosix.kt index fd053639ad..00e5a633f3 100644 --- a/ktor-shared/ktor-test-base/jvmAndPosix/src/io/ktor/test/TestResult.jvmAndPosix.kt +++ b/ktor-shared/ktor-test-base/jvmAndPosix/src/io/ktor/test/TestResult.jvmAndPosix.kt @@ -1,5 +1,5 @@ /* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. */ package io.ktor.test @@ -25,7 +25,7 @@ internal actual inline fun runTestForEach(items: Iterable, test: (T) -> T return DummyTestResult } -internal actual inline fun retryTest(retries: Int, test: (Int) -> TestResult): TestResult { +actual inline fun retryTest(retries: Int, test: (Int) -> TestResult): TestResult { lateinit var lastCause: Throwable repeat(retries + 1) { attempt -> try {