KTOR-8043 Retry server tests one time by default

ktorio · Jan 10, 2025 · e7c9218 · e7c9218
1 parent b42d338
commit e7c9218
Show file tree

Hide file tree

Showing 10 changed files with 79 additions and 100 deletions.
diff --git a/ktor-server/ktor-server-test-base/build.gradle.kts b/ktor-server/ktor-server-test-base/build.gradle.kts
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 description = ""
@@ -10,7 +10,7 @@ kotlin.sourceSets {
     commonMain {
         dependencies {
             api(project(":ktor-server:ktor-server-test-host"))
-            api(libs.kotlin.test)
+            api(project(":ktor-shared:ktor-test-base"))
         }
     }
 
@@ -21,7 +21,6 @@ kotlin.sourceSets {
             api(project(":ktor-client:ktor-client-apache"))
             api(project(":ktor-network:ktor-network-tls:ktor-network-tls-certificates"))
             api(project(":ktor-server:ktor-server-plugins:ktor-server-call-logging"))
-            api(project(":ktor-shared:ktor-test-base"))
 
             if (jetty_alpn_boot_version != null) {
                 api(libs.jetty.alpn.boot)

diff --git a/ktor-server/ktor-server-test-base/common/src/io/ktor/server/test/base/BaseTest.kt b/ktor-server/ktor-server-test-base/common/src/io/ktor/server/test/base/BaseTest.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.server.test.base
@@ -16,21 +16,15 @@ expect abstract class BaseTest() {
     open fun afterTest()
 
     fun collectUnhandledException(error: Throwable) // TODO: better name?
-    fun runTest(timeout: Duration = 60.seconds, block: suspend CoroutineScope.() -> Unit): TestResult
+    fun runTest(
+        timeout: Duration = 60.seconds,
+        retries: Int = DEFAULT_RETRIES,
+        block: suspend CoroutineScope.() -> Unit
+    ): TestResult
 }
 
-fun BaseTest.runTest(
-    retry: Int,
-    timeout: Duration = this.timeout,
-    block: suspend CoroutineScope.() -> Unit
-): TestResult {
-    lateinit var lastCause: Throwable
-    repeat(retry) {
-        try {
-            return runTest(timeout, block)
-        } catch (cause: Throwable) {
-            lastCause = cause
-        }
-    }
-    throw lastCause
-}
+/**
+ * Defaults to `1` on all platforms except for JVM.
+ * On JVM retries are disabled as we use test-retry Gradle plugin instead.
+ */
+internal expect val DEFAULT_RETRIES: Int
diff --git a/...server-test-base/jsAndWasmShared/src/io/ktor/server/test/base/BaseTest.jsAndWasmShared.kt b/...server-test-base/jsAndWasmShared/src/io/ktor/server/test/base/BaseTest.jsAndWasmShared.kt
diff --git a/ktor-server/ktor-server-test-base/jvm/src/io/ktor/server/test/base/BaseTestJvm.kt b/ktor-server/ktor-server-test-base/jvm/src/io/ktor/server/test/base/BaseTestJvm.kt
@@ -1,9 +1,10 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.server.test.base
 
+import io.ktor.test.*
 import io.ktor.test.dispatcher.*
 import io.ktor.test.junit.*
 import io.ktor.test.junit.coroutines.*
@@ -39,6 +40,7 @@ actual abstract class BaseTest actual constructor() {
     }
 
     actual open fun beforeTest() {
+        errorCollector.clear()
     }
 
     actual fun collectUnhandledException(error: Throwable) {
@@ -47,13 +49,19 @@ actual abstract class BaseTest actual constructor() {
 
     actual fun runTest(
         timeout: Duration,
+        retries: Int,
         block: suspend CoroutineScope.() -> Unit
-    ): TestResult = runTestWithRealTime(CoroutineName("test-$testName"), timeout) {
-        beforeTest()
-        try {
-            block()
-        } finally {
-            afterTest()
+    ): TestResult = retryTest(retries) {
+        runTestWithRealTime(CoroutineName("test-$testName"), timeout) {
+            beforeTest()
+            try {
+                block()
+            } finally {
+                afterTest()
+            }
         }
     }
 }
+
+/** On JVM retries are disabled as we use test-retry Gradle plugin instead. */
+internal actual const val DEFAULT_RETRIES: Int = 0
diff --git a/...c/io/ktor/server/test/base/BaseTestNix.kt → .../ktor/server/test/base/BaseTest.nonJvm.kt b/...c/io/ktor/server/test/base/BaseTestNix.kt → .../ktor/server/test/base/BaseTest.nonJvm.kt
@@ -1,9 +1,10 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.server.test.base
 
+import io.ktor.test.*
 import io.ktor.test.dispatcher.*
 import io.ktor.utils.io.*
 import io.ktor.utils.io.locks.*
@@ -12,22 +13,24 @@ import kotlinx.coroutines.test.TestResult
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.seconds
 
+@OptIn(InternalAPI::class)
 actual abstract class BaseTest actual constructor() {
     actual open val timeout: Duration = 10.seconds
 
     private val errors = mutableListOf<Throwable>()
 
-    @OptIn(InternalAPI::class)
     private val errorsLock = SynchronizedObject()
 
-    @OptIn(InternalAPI::class)
     actual fun collectUnhandledException(error: Throwable) {
         synchronized(errorsLock) {
             errors.add(error)
         }
     }
 
     actual open fun beforeTest() {
+        synchronized(errorsLock) {
+            errors.clear()
+        }
     }
 
     actual open fun afterTest() {
@@ -46,15 +49,20 @@ actual abstract class BaseTest actual constructor() {
 
     actual fun runTest(
         timeout: Duration,
+        retries: Int,
         block: suspend CoroutineScope.() -> Unit
-    ): TestResult = runTestWithRealTime(timeout = timeout) {
-        beforeTest()
-        try {
-            block()
-        } finally {
-            afterTest()
+    ): TestResult = retryTest(retries) {
+        runTestWithRealTime(timeout = timeout) {
+            beforeTest()
+            try {
+                block()
+            } finally {
+                afterTest()
+            }
         }
     }
 }
 
+internal actual const val DEFAULT_RETRIES: Int = 1
+
 private class UnhandledErrorsException(override val message: String) : Exception()
diff --git a/ktor-shared/ktor-test-base/common/src/io/ktor/test/TestResult.kt b/ktor-shared/ktor-test-base/common/src/io/ktor/test/TestResult.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test
@@ -18,4 +18,21 @@ expect inline fun TestResult.andThen(crossinline block: () -> Any): TestResult
 internal expect inline fun testWithRecover(noinline recover: (Throwable) -> Unit, test: () -> TestResult): TestResult
 
 internal expect inline fun <T> runTestForEach(items: Iterable<T>, crossinline test: (T) -> TestResult): TestResult
-internal expect inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult
+
+/**
+ * Executes a test function with retry capabilities.
+ *
+ * ```
+ * retryTest(retires = 2) { retry ->
+ *     runTest {
+ *         println("This test passes only on second retry. Current retry is $retry")
+ *         assertEquals(2, retry)
+ *     }
+ * }
+ * ```
+ *
+ * @param retries The number of retries to attempt after an initial failure. Must be a non-negative integer.
+ * @param test A test to execute, which accepts the current retry attempt (starting at 0) as an argument.
+ * @return A [TestResult] representing the outcome of the test after all attempts.
+ */
+expect inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult
diff --git a/ktor-shared/ktor-test-base/common/test/io/ktor/test/RunTestWithDataTest.kt b/ktor-shared/ktor-test-base/common/test/io/ktor/test/RunTestWithDataTest.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test
@@ -101,18 +101,18 @@ class RunTestWithDataTest {
     fun testRetriesHaveIndependentTimeout() = runTestWithData(
         singleTestCase,
         retries = 1,
-        timeout = 30.milliseconds,
+        timeout = 50.milliseconds,
         test = { (_, retry) ->
-            realTimeDelay(20.milliseconds)
+            realTimeDelay(30.milliseconds)
             if (retry == 0) fail("Try again, please")
         },
     )
 
     @Test
     fun testDifferentItemsHaveIndependentTimeout() = runTestWithData(
         testCases = 1..2,
-        timeout = 30.milliseconds,
-        test = { realTimeDelay(20.milliseconds) },
+        timeout = 50.milliseconds,
+        test = { realTimeDelay(30.milliseconds) },
     )
 
     @Test

diff --git a/ktor-shared/ktor-test-base/jsAndWasmShared/src/io/ktor/test/TestResult.jsAndWasmShared.kt b/ktor-shared/ktor-test-base/jsAndWasmShared/src/io/ktor/test/TestResult.jsAndWasmShared.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test
@@ -14,7 +14,8 @@ internal actual inline fun testWithRecover(
 internal actual inline fun <T> runTestForEach(items: Iterable<T>, crossinline test: (T) -> TestResult): TestResult =
     items.fold(DummyTestResult) { acc, item -> acc.andThen { test(item) } }
 
-internal actual inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult =
+actual inline fun retryTest(retries: Int, crossinline test: (Int) -> TestResult): TestResult =
     (1..retries).fold(test(0)) { acc, retry -> acc.catch { test(retry) } }
 
+@PublishedApi
 internal expect fun TestResult.catch(action: (Throwable) -> Any): TestResult
diff --git a/ktor-shared/ktor-test-base/jvm/src/io/ktor/test/junit/ErrorCollector.kt b/ktor-shared/ktor-test-base/jvm/src/io/ktor/test/junit/ErrorCollector.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test.junit
@@ -50,4 +50,8 @@ class ErrorCollector : TestExecutionExceptionHandler, ParameterResolver, AfterEa
             }
         }
     }
+
+    fun clear() {
+        errors.clear()
+    }
 }
diff --git a/ktor-shared/ktor-test-base/jvmAndPosix/src/io/ktor/test/TestResult.jvmAndPosix.kt b/ktor-shared/ktor-test-base/jvmAndPosix/src/io/ktor/test/TestResult.jvmAndPosix.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2014-2025 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
  */
 
 package io.ktor.test
@@ -25,7 +25,7 @@ internal actual inline fun <T> runTestForEach(items: Iterable<T>, test: (T) -> T
     return DummyTestResult
 }
 
-internal actual inline fun retryTest(retries: Int, test: (Int) -> TestResult): TestResult {
+actual inline fun retryTest(retries: Int, test: (Int) -> TestResult): TestResult {
     lateinit var lastCause: Throwable
     repeat(retries + 1) { attempt ->
         try {