feat components: add graceful_shutdown_interval

commit_hash:48caf2b51711c92db14553f07360d152f11581ca

.mapping.json

@@ -502,6 +502,12 @@
   "core/functional_tests/dynamic_configs/tests/test_changelog.py":"taxi/uservices/userver/core/functional_tests/dynamic_configs/tests/test_changelog.py",
   "core/functional_tests/dynamic_configs/tests/test_fixtures.py":"taxi/uservices/userver/core/functional_tests/dynamic_configs/tests/test_fixtures.py",
   "core/functional_tests/dynamic_configs/tests/test_invalid_update.py":"taxi/uservices/userver/core/functional_tests/dynamic_configs/tests/test_invalid_update.py",
+  "core/functional_tests/graceful_shutdown/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/CMakeLists.txt",
+  "core/functional_tests/graceful_shutdown/main.cpp":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/main.cpp",
+  "core/functional_tests/graceful_shutdown/src/handler_sigterm.hpp":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/src/handler_sigterm.hpp",
+  "core/functional_tests/graceful_shutdown/static_config.yaml":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/static_config.yaml",
+  "core/functional_tests/graceful_shutdown/tests/conftest.py":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/tests/conftest.py",
+  "core/functional_tests/graceful_shutdown/tests/test_graceful_shutdown.py":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/tests/test_graceful_shutdown.py",
   "core/functional_tests/http2server/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/http2server/CMakeLists.txt",
   "core/functional_tests/http2server/service.cpp":"taxi/uservices/userver/core/functional_tests/http2server/service.cpp",
   "core/functional_tests/http2server/static_config.yaml":"taxi/uservices/userver/core/functional_tests/http2server/static_config.yaml",

core/functional_tests/CMakeLists.txt

@@ -8,6 +8,9 @@ add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-basic-chaos)
 add_subdirectory(dynamic_configs)
 add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-dynamic-configs)
 
+add_subdirectory(graceful_shutdown)
+add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-graceful-shutdown)
+
 add_subdirectory(https)
 add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-https)
 

core/functional_tests/basic_chaos/tests-restart/conftest.py

@@ -1 +1 @@
-pytest_plugins = ['pytest_userver.plugins.core', 'pytest_userver.plugins']
+pytest_plugins = ['pytest_userver.plugins.core']

core/functional_tests/basic_chaos/tests-restart/test_restart.py

@@ -23,4 +23,4 @@ async def test_restart(monitor_client, service_client, _global_daemon_store):
     response = await service_client.get('/ping')
     assert response.status == 500
 
-    wait_for_daemon_stop(_global_daemon_store)
+    await wait_for_daemon_stop(_global_daemon_store)

core/functional_tests/graceful_shutdown/CMakeLists.txt

@@ -0,0 +1,8 @@
+project(userver-core-tests-graceful-shutdown CXX)
+
+file(GLOB_RECURSE SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*pp")
+add_executable(${PROJECT_NAME} ${SOURCES} "main.cpp")
+target_include_directories(${PROJECT_NAME} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
+target_link_libraries(${PROJECT_NAME} userver-core)
+
+userver_chaos_testsuite_add()

core/functional_tests/graceful_shutdown/main.cpp

@@ -0,0 +1,12 @@
+#include <userver/components/minimal_server_component_list.hpp>
+#include <userver/server/handlers/ping.hpp>
+#include <userver/utest/using_namespace_userver.hpp>
+#include <userver/utils/daemon_run.hpp>
+
+#include <handler_sigterm.hpp>
+
+int main(int argc, char* argv[]) {
+    const auto component_list =
+        components::MinimalServerComponentList().Append<server::handlers::Ping>().Append<handlers::Sigterm>();
+    return utils::DaemonMain(argc, argv, component_list);
+}

core/functional_tests/graceful_shutdown/src/handler_sigterm.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <signal.h>
+#include <unistd.h>
+
+#include <userver/server/handlers/http_handler_base.hpp>
+#include <userver/utest/using_namespace_userver.hpp>
+
+namespace handlers {
+
+class Sigterm final : public server::handlers::HttpHandlerBase {
+public:
+    static constexpr std::string_view kName = "handler-sigterm";
+
+    Sigterm(const components::ComponentConfig& config, const components::ComponentContext& context)
+        : HttpHandlerBase(config, context) {}
+
+    std::string HandleRequestThrow(const server::http::HttpRequest& /*request*/, server::request::RequestContext&)
+        const override {
+        kill(getpid(), SIGTERM);
+        return {};
+    }
+};
+
+}  // namespace handlers

core/functional_tests/graceful_shutdown/static_config.yaml

@@ -0,0 +1,38 @@
+components_manager:
+    task_processors:
+        main-task-processor:
+            worker_threads: 4
+
+        fs-task-processor:
+            worker_threads: 2
+
+    default_task_processor: main-task-processor
+
+    components:
+        logging:
+            fs-task-processor: fs-task-processor
+            loggers:
+                default:
+                    file_path: '@stderr'
+                    level: debug
+                    overflow_behavior: discard
+
+        server:
+            listener:
+                port: 8080
+                task_processor: main-task-processor
+            listener-monitor:
+                port: 8081
+                task_processor: main-task-processor
+
+        handler-ping:
+            path: /ping
+            method: GET
+            task_processor: main-task-processor
+            throttling_enabled: false
+
+        handler-sigterm:
+            monitor-handler: true
+            path: /sigterm
+            method: POST
+            task_processor: main-task-processor

core/functional_tests/graceful_shutdown/tests/conftest.py

@@ -0,0 +1,13 @@
+import pytest
+
+pytest_plugins = ['pytest_userver.plugins.core']
+
+
+# Overriding a userver fixture
+@pytest.fixture(name='userver_config_testsuite', scope='session')
+def _userver_config_testsuite(userver_config_testsuite):
+    def patch_config(config, config_vars) -> None:
+        userver_config_testsuite(config, config_vars)
+        config['components_manager']['graceful_shutdown_interval'] = '3s'
+
+    return patch_config

core/functional_tests/graceful_shutdown/tests/test_graceful_shutdown.py

@@ -0,0 +1,37 @@
+import asyncio
+import datetime
+
+
+async def wait_for_daemon_stop(_global_daemon_store):
+    deadline = datetime.datetime.now() + datetime.timedelta(seconds=10)
+    while (
+        datetime.datetime.now() < deadline
+        and _global_daemon_store.has_running_daemons()
+    ):
+        await asyncio.sleep(0.05)
+
+    assert (
+        not _global_daemon_store.has_running_daemons()
+    ), 'Daemon has not stopped'
+    await _global_daemon_store.aclose()
+
+
+async def test_graceful_shutdown_timer(
+    service_client, monitor_client, _global_daemon_store,
+):
+    response = await service_client.get('/ping')
+    assert response.status == 200
+
+    response = await monitor_client.post('/sigterm')
+    assert response.status == 200
+
+    response = await service_client.get('/ping')
+    assert response.status == 500
+
+    await asyncio.sleep(2)
+    # Check that the service is still alive.
+    response = await service_client.get('/ping')
+    assert response.status == 500
+
+    # After a couple more seconds, the service will start shutting down.
+    await wait_for_daemon_stop(_global_daemon_store)

core/include/userver/components/component_context.hpp

@@ -178,6 +178,8 @@ class ComponentContext final {
 
     void OnAllComponentsLoaded();
 
+    void OnGracefulShutdownStarted();
+
     void OnAllComponentsAreStopping();
 
     void ClearComponents();

core/src/components/component_context.cpp

@@ -28,6 +28,8 @@ RawComponentBase* ComponentContext::AddComponent(std::string_view name, const im
 
 void ComponentContext::OnAllComponentsLoaded() { impl_->OnAllComponentsLoaded(); }
 
+void ComponentContext::OnGracefulShutdownStarted() { impl_->OnGracefulShutdownStarted(); }
+
 void ComponentContext::OnAllComponentsAreStopping() { impl_->OnAllComponentsAreStopping(); }
 
 void ComponentContext::ClearComponents() { impl_->ClearComponents(); }

core/src/components/component_context_impl.cpp

@@ -7,6 +7,7 @@
 
 #include <userver/compiler/demangle.hpp>
 #include <userver/concurrent/variable.hpp>
+#include <userver/engine/sleep.hpp>
 #include <userver/engine/task/task_with_result.hpp>
 #include <userver/logging/log.hpp>
 #include <userver/tracing/tracer.hpp>
@@ -16,6 +17,7 @@
 #include <components/component_context_component_info.hpp>
 #include <components/impl/component_name_from_info.hpp>
 #include <components/manager.hpp>
+#include <components/manager_config.hpp>
 #include <engine/task/task_context.hpp>
 
 USERVER_NAMESPACE_BEGIN
@@ -136,6 +138,16 @@ void ComponentContextImpl::OnAllComponentsLoaded() {
     );
 }
 
+void ComponentContextImpl::OnGracefulShutdownStarted() {
+    shutdown_started_ = true;
+
+    const auto interval = manager_.GetConfig().graceful_shutdown_interval;
+    if (interval > std::chrono::milliseconds{0}) {
+        LOG_INFO() << "Shutdown started, notifying ping handlers and delaying by " << interval;
+        engine::SleepFor(interval);
+    }
+}
+
 void ComponentContextImpl::OnAllComponentsAreStopping() {
     LOG_INFO() << "Sending stopping notification to all components";
     ProcessAllComponentLifetimeStageSwitchings(
@@ -209,6 +221,11 @@ bool ComponentContextImpl::IsAnyComponentInFatalState() const {
         }
     }
 
+    if (shutdown_started_) {
+        LOG_WARNING() << "Service is shutting down, returning 5xx from ping";
+        return true;
+    }
+
     return false;
 }
 

core/src/components/component_context_impl.hpp

@@ -31,6 +31,8 @@ class ComponentContextImpl {
 
     void OnAllComponentsLoaded();
 
+    void OnGracefulShutdownStarted();
+
     void OnAllComponentsAreStopping();
 
     void ClearComponents();
@@ -155,6 +157,8 @@ class ComponentContextImpl {
     engine::ConditionVariable print_adding_components_cv_;
     concurrent::Variable<ProtectedData> shared_data_;
     engine::TaskWithResult<void> print_adding_components_task_;
+
+    std::atomic<bool> shutdown_started_{false};
 };
 
 }  // namespace components::impl

core/src/components/manager.cpp

@@ -191,6 +191,11 @@ Manager::Manager(std::unique_ptr<ManagerConfig>&& config, const ComponentList& c
 Manager::~Manager() {
     LOG_INFO() << "Stopping components manager";
 
+    try {
+        RunInCoro(*default_task_processor_, [this] { component_context_.OnGracefulShutdownStarted(); });
+    } catch (const std::exception& exc) {
+        LOG_ERROR() << "Graceful shutdown failed: " << exc;
+    }
     engine::impl::TeardownPhdrCacheAndEnableDynamicLoading();
 
     LOG_TRACE() << "Stopping component context";

core/src/components/manager_config.cpp

@@ -221,6 +221,13 @@ additionalProperties: false
         additionalProperties:
             type: boolean
             description: whether a specific experiment is enabled
+    graceful_shutdown_interval:
+        type: string
+        description: |
+            At shutdown, first hang for this duration with /ping 5xx to give
+            the balancer a chance to redirect new requests to other hosts and
+            to give the service a chance to finish handling old requests.
+        defaultDescription: 0s
 )");
 }
 
@@ -251,6 +258,9 @@ ManagerConfig Parse(const yaml_config::YamlConfig& value, formats::parse::To<Man
     config.disable_phdr_cache = value["disable_phdr_cache"].As<bool>(config.disable_phdr_cache);
     config.preheat_stacktrace_collector =
         value["preheat_stacktrace_collector"].As<bool>(config.preheat_stacktrace_collector);
+    config.graceful_shutdown_interval =
+        value["graceful_shutdown_interval"].As<std::chrono::milliseconds>(config.graceful_shutdown_interval);
+
     return config;
 }
 

core/src/components/manager_config.hpp

@@ -25,6 +25,7 @@ struct ManagerConfig {
     std::string default_task_processor;
     ValidationMode validate_components_configs{};
     utils::impl::UserverExperimentSet enabled_experiments;
+    std::chrono::milliseconds graceful_shutdown_interval{};
     bool mlock_debug_info{true};
     bool disable_phdr_cache{false};
     bool preheat_stacktrace_collector{true};

testsuite/pytest_plugins/pytest_userver/plugins/config.py

@@ -504,6 +504,8 @@ def _disable_cache_periodic_update(testsuite_support: dict) -> None:
         testsuite_support['testsuite-periodic-update-enabled'] = False
 
     def patch_config(config, config_vars) -> None:
+        # Don't delay tests teardown unnecessarily.
+        config['components_manager'].pop('graceful_shutdown_interval', None)
         components: dict = config['components_manager']['components']
         if 'testsuite-support' not in components:
             return