fix: Provide a way to customize the graceful shutdown delay of Netty EventLoopGroups

core/src/main/java/com/linecorp/armeria/client/ClientFactoryBuilder.java

@@ -926,6 +926,54 @@ public ClientFactoryBuilder connectionOutlierDetection(OutlierDetection outlierD
         return this;
     }
 
+    /**
+     * Sets the quiet period in milliseconds for graceful shutdown of a worker group.
+     * {@code 0} removes the quiet period entirely.
+     */
+    @UnstableApi
+    public ClientFactoryBuilder workerGroupGracefulShutdownQuietPeriod(Duration duration) {
+        requireNonNull(duration, "duration");
+        return workerGroupGracefulShutdownQuietPeriodMillis(duration.toMillis());
+    }
+
+    /**
+     * Sets the quiet period in milliseconds for graceful shutdown of a worker group.
+     * {@code 0} removes the quiet period entirely.
+     */
+    @UnstableApi
+    public ClientFactoryBuilder workerGroupGracefulShutdownQuietPeriodMillis(long workerGroupGracefulShutdownQuietPeriodMillis) {
+        checkArgument(workerGroupGracefulShutdownQuietPeriodMillis >= 0,
+                      "workerGroupGracefulShutdownQuietPeriodMillis: %s (expected: >= 0)",
+                      workerGroupGracefulShutdownQuietPeriodMillis);
+        option(ClientFactoryOptions.WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS,
+               workerGroupGracefulShutdownQuietPeriodMillis);
+        return this;
+    }
+
+    /**
+     * Sets the timeout in milliseconds for graceful shutdown of a worker group.
+     * {@code 0} disables the timeout and closes the worker group immediately.
+     */
+    @UnstableApi
+    public ClientFactoryBuilder workerGroupGracefulShutdownTimeout(Duration duration) {
+        requireNonNull(duration, "duration");
+        return workerGroupGracefulShutdownTimeoutMillis(duration.toMillis());
+    }
+
+    /**
+     * Sets the timeout in milliseconds for graceful shutdown of a worker group.
+     * {@code 0} disables the timeout and closes the worker group immediately.
+     */
+    @UnstableApi
+    public ClientFactoryBuilder workerGroupGracefulShutdownTimeoutMillis(long workerGroupGracefulShutdownTimeoutMillis) {
+        checkArgument(workerGroupGracefulShutdownTimeoutMillis >= 0,
+                      "workerGroupGracefulShutdownTimeoutMillis: %s (expected: >= 0)",
+                      workerGroupGracefulShutdownTimeoutMillis);
+        option(ClientFactoryOptions.WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS,
+               workerGroupGracefulShutdownTimeoutMillis);
+        return this;
+    }
+
     /**
      * Sets the graceful connection shutdown timeout in milliseconds.
      * {@code 0} disables the timeout and closes the connection immediately after sending a GOAWAY frame.

core/src/main/java/com/linecorp/armeria/client/ClientFactoryOptions.java

@@ -305,7 +305,21 @@ private static long clampedDefaultMaxClientConnectionAge() {
             ClientFactoryOption.define("CONNECTION_POOL_LISTENER", ConnectionPoolListener.noop());
 
     /**
-     * The graceful connection shutdown timeout in milliseconds..
+     * The graceful worker group pool quiet period in milliseconds.
+     */
+    public static final ClientFactoryOption<Long> WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS =
+            ClientFactoryOption.define("WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS",
+                                       Flags.defaultClientWorkerGroupGracefulShutdownQuietPeriodMillis());
+
+    /**
+     * The graceful worker group pool time out in milliseconds.
+     */
+    public static final ClientFactoryOption<Long> WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS =
+            ClientFactoryOption.define("WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS",
+                                       Flags.defaultClientWorkerGroupGracefulShutdownTimeoutMillis());
+
+    /**
+     * The graceful connection shutdown timeout in milliseconds.
      */
     public static final ClientFactoryOption<Long> HTTP2_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS =
             ClientFactoryOption.define("HTTP2_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS",
@@ -661,6 +675,20 @@ public OutlierDetection connectionOutlierDetection() {
         return get(CONNECTION_OUTLIER_DETECTION);
     }
 
+    /**
+     * Returns the graceful worker group pool quiet period in milliseconds.
+     */
+    public long workerGroupGracefulShutdownQuietPeriodMillis() {
+        return get(WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS);
+    }
+
+    /**
+     * Returns the graceful worker group pool time out in milliseconds.
+     */
+    public long workerGroupGracefulShutdownTimeoutMillis() {
+        return get(WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS);
+    }
+
     /**
      * Returns the graceful connection shutdown timeout in milliseconds.
      */

core/src/main/java/com/linecorp/armeria/client/HttpClientFactory.java

@@ -28,6 +28,7 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
@@ -494,7 +495,11 @@ private void closeAsync(CompletableFuture<?> future) {
                 connectionPoolListener.close();
             }
             if (shutdownWorkerGroupOnClose) {
-                workerGroup.shutdownGracefully().addListener((FutureListener<Object>) f -> {
+                workerGroup.shutdownGracefully(
+                        options.workerGroupGracefulShutdownQuietPeriodMillis(),
+                        options.workerGroupGracefulShutdownTimeoutMillis(),
+                        TimeUnit.MILLISECONDS
+                ).addListener((FutureListener<Object>) f -> {
                     if (f.cause() != null) {
                         logger.warn("Failed to shut down a worker group:", f.cause());
                     }

core/src/main/java/com/linecorp/armeria/common/DefaultFlagsProvider.java

@@ -81,6 +81,10 @@ final class DefaultFlagsProvider implements FlagsProvider {
     static final long DEFAULT_SERVER_CONNECTION_DRAIN_DURATION_MICROS = 1000000;
     // Same as server connection drain duration
     static final long DEFAULT_CLIENT_HTTP2_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS = 1000;
+    // Same as AbstractEventExecutor.DEFAULT_SHUTDOWN_QUIET_PERIOD
+    static final long DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS = 15000L;
+    // Same as AbstractEventExecutor.DEFAULT_SHUTDOWN_TIMEOUT
+    static final long DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS = 2000L;
     static final int DEFAULT_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE = 1024 * 1024; // 1MiB
     static final int DEFAULT_HTTP2_INITIAL_STREAM_WINDOW_SIZE = 1024 * 1024; // 1MiB
     static final float DEFAULT_HTTP2_STREAM_WINDOW_UPDATE_RATIO =
@@ -319,6 +323,16 @@ public Long defaultServerConnectionDrainDurationMicros() {
         return DEFAULT_SERVER_CONNECTION_DRAIN_DURATION_MICROS;
     }
 
+    @Override
+    public Long defaultClientWorkerGroupGracefulShutdownQuietPeriodMillis() {
+        return DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS;
+    }
+
+    @Override
+    public Long defaultClientWorkerGroupGracefulShutdownTimeoutMillis() {
+        return DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS;
+    }
+
     @Override
     public Long defaultClientHttp2GracefulShutdownTimeoutMillis() {
         return DEFAULT_CLIENT_HTTP2_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS;

core/src/main/java/com/linecorp/armeria/common/Flags.java

@@ -291,6 +291,14 @@ private static boolean validateTransportType(TransportType transportType, String
             getValue(FlagsProvider::defaultServerConnectionDrainDurationMicros,
                      "defaultServerConnectionDrainDurationMicros", value -> value >= 0);
 
+    private static final long DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS =
+            getValue(FlagsProvider::defaultClientWorkerGroupGracefulShutdownQuietPeriodMillis,
+                     "defaultClientWorkerGroupGracefulShutdownQuietPeriodMillis", value -> value >= 0);
+
+    private static final long DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS =
+            getValue(FlagsProvider::defaultClientWorkerGroupGracefulShutdownTimeoutMillis,
+                     "defaultClientWorkerGroupGracefulShutdownTimeoutMillis", value -> value >= 0);
+
     private static final long DEFAULT_CLIENT_HTTP2_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS =
             getValue(FlagsProvider::defaultClientHttp2GracefulShutdownTimeoutMillis,
                      "defaultClientHttp2GracefulShutdownTimeoutMillis", value -> value >= 0);
@@ -1076,6 +1084,42 @@ public static long defaultServerConnectionDrainDurationMicros() {
         return DEFAULT_SERVER_CONNECTION_DRAIN_DURATION_MICROS;
     }
 
+    /**
+     * Returns the default client-side graceful worker group shutdown quiet period in milliseconds.
+     * {@code 0} disables the quiet period.
+     *
+     * <p>The default value of this flag is
+     * {@value DefaultFlagsProvider#DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS}.
+     * Specify the {@code -Dcom.linecorp.armeria.defaultClientWorkerGroupGracefulShutdownQuietPeriodMillis=<long>}
+     * JVM option to override the default value. {@code 0} disables the graceful shutdown.
+     * </p>
+     *
+     * @see ClientFactoryBuilder#workerGroupGracefulShutdownQuietPeriod(Duration)
+     * @see ClientFactoryBuilder#workerGroupGracefulShutdownQuietPeriodMillis(long)
+     */
+    @UnstableApi
+    public static long defaultClientWorkerGroupGracefulShutdownQuietPeriodMillis() {
+        return DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS;
+    }
+
+    /**
+     * Returns the default client-side graceful worker group shutdown timeout in milliseconds.
+     * {@code 0} disables the timeout and closes the worker group immediately.
+     *
+     * <p>The default value of this flag is
+     * {@value DefaultFlagsProvider#DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS}.
+     * Specify the {@code -Dcom.linecorp.armeria.defaultClientWorkerGroupGracefulShutdownTimeoutMillis=<long>}
+     * JVM option to override the default value. {@code 0} disables the graceful shutdown.
+     * </p>
+     *
+     * @see ClientFactoryBuilder#workerGroupGracefulShutdownTimeout(Duration)
+     * @see ClientFactoryBuilder#workerGroupGracefulShutdownTimeoutMillis(long)
+     */
+    @UnstableApi
+    public static long defaultClientWorkerGroupGracefulShutdownTimeoutMillis() {
+        return DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS;
+    }
+
     /**
      * Returns the default client-side graceful connection shutdown timeout in milliseconds.
      * {@code 0} disables the timeout and closes the connection immediately after sending a GOAWAY frame.

core/src/main/java/com/linecorp/armeria/common/FlagsProvider.java

@@ -588,7 +588,39 @@ default Integer defaultMaxClientNumRequestsPerConnection() {
     }
 
     /**
-     * Returns the default client-side graceful connection shutdown timeout in microseconds.
+     * Returns the default client-side graceful worker group pool shutdown quiet period in milliseconds.
+     *
+     * <p>Note that this flag has no effect if a user specified the value explicitly via
+     * {@link ClientFactoryBuilder#workerGroupGracefulShutdownQuietPeriodMillis(long)}.
+     *
+     * <p>The default value of this flag is
+     * {@value DefaultFlagsProvider#DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_QUIET_PERIOD_MILLIS}.
+     * Specify the {@code -Dcom.linecorp.armeria.defaultClientWorkerGroupGracefulShutdownQuietPeriodMillis=<long>}
+     * JVM option to override the default value. {@code 0} disables the graceful shutdown.
+     */
+    @Nullable
+    default Long defaultClientWorkerGroupGracefulShutdownQuietPeriodMillis() {
+        return null;
+    }
+
+    /**
+     * Returns the default client-side graceful worker group pool shutdown timeout in milliseconds.
+     *
+     * <p>Note that this flag has no effect if a user specified the value explicitly via
+     * {@link ClientFactoryBuilder#workerGroupGracefulShutdownTimeoutMillis(long)}.
+     *
+     * <p>The default value of this flag is
+     * {@value DefaultFlagsProvider#DEFAULT_CLIENT_WORKER_GROUP_GRACEFUL_SHUTDOWN_TIMEOUT_MILLIS}.
+     * Specify the {@code -Dcom.linecorp.armeria.defaultClientWorkerGroupGracefulShutdownTimeoutMillis=<long>}
+     * JVM option to override the default value. {@code 0} disables the graceful shutdown.
+     */
+    @Nullable
+    default Long defaultClientWorkerGroupGracefulShutdownTimeoutMillis() {
+        return null;
+    }
+
+    /**
+     * Returns the default client-side graceful connection shutdown timeout in milliseconds.
      *
      * <p>Note that this flag has no effect if a user specified the value explicitly via
      * {@link ClientFactoryBuilder#http2GracefulShutdownTimeoutMillis(long)}.

core/src/main/java/com/linecorp/armeria/server/Server.java

@@ -690,7 +690,12 @@ private void doStop(CompletableFuture<Void> future,
                         // Shut down the worker group if necessary.
                         final Future<?> workerShutdownFuture;
                         if (config.shutdownWorkerGroupOnStop()) {
-                            workerShutdownFuture = config.workerGroup().shutdownGracefully();
+                            final GracefulShutdown gracefulShutdown = config().gracefulShutdown();
+                            workerShutdownFuture = config.workerGroup().shutdownGracefully(
+                                gracefulShutdown.quietPeriod().toMillis(),
+                                gracefulShutdown.timeout().toMillis(),
+                                TimeUnit.MILLISECONDS
+                            );
                         } else {
                             workerShutdownFuture = ImmediateEventExecutor.INSTANCE.newSucceededFuture(null);
                         }
@@ -711,7 +716,12 @@ private void doStop(CompletableFuture<Void> future,
                             // Shut down all boss groups and wait until they are terminated.
                             final AtomicInteger remainingBossGroups = new AtomicInteger(bossGroups.size());
                             bossGroups.forEach(bossGroup -> {
-                                bossGroup.shutdownGracefully();
+                                final GracefulShutdown gracefulShutdown = config().gracefulShutdown();
+                                bossGroup.shutdownGracefully(
+                                    gracefulShutdown.quietPeriod().toMillis(),
+                                    gracefulShutdown.timeout().toMillis(),
+                                    TimeUnit.MILLISECONDS
+                                );
                                 bossGroup.terminationFuture().addListener(unused8 -> {
                                     if (remainingBossGroups.decrementAndGet() != 0) {
                                         // There are more boss groups to terminate.