diff --git a/.mapping.json b/.mapping.json index b3eafad8270e..f6e0c2cb6507 100644 --- a/.mapping.json +++ b/.mapping.json @@ -502,6 +502,12 @@ "core/functional_tests/dynamic_configs/tests/test_changelog.py":"taxi/uservices/userver/core/functional_tests/dynamic_configs/tests/test_changelog.py", "core/functional_tests/dynamic_configs/tests/test_fixtures.py":"taxi/uservices/userver/core/functional_tests/dynamic_configs/tests/test_fixtures.py", "core/functional_tests/dynamic_configs/tests/test_invalid_update.py":"taxi/uservices/userver/core/functional_tests/dynamic_configs/tests/test_invalid_update.py", + "core/functional_tests/graceful_shutdown/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/CMakeLists.txt", + "core/functional_tests/graceful_shutdown/main.cpp":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/main.cpp", + "core/functional_tests/graceful_shutdown/src/handler_sigterm.hpp":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/src/handler_sigterm.hpp", + "core/functional_tests/graceful_shutdown/static_config.yaml":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/static_config.yaml", + "core/functional_tests/graceful_shutdown/tests/conftest.py":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/tests/conftest.py", + "core/functional_tests/graceful_shutdown/tests/test_graceful_shutdown.py":"taxi/uservices/userver/core/functional_tests/graceful_shutdown/tests/test_graceful_shutdown.py", "core/functional_tests/http2server/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/http2server/CMakeLists.txt", "core/functional_tests/http2server/service.cpp":"taxi/uservices/userver/core/functional_tests/http2server/service.cpp", "core/functional_tests/http2server/static_config.yaml":"taxi/uservices/userver/core/functional_tests/http2server/static_config.yaml", diff --git a/core/functional_tests/CMakeLists.txt b/core/functional_tests/CMakeLists.txt index dc5c9462e94d..626ba088f457 100644 --- a/core/functional_tests/CMakeLists.txt +++ b/core/functional_tests/CMakeLists.txt @@ -8,6 +8,9 @@ add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-basic-chaos) add_subdirectory(dynamic_configs) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-dynamic-configs) +add_subdirectory(graceful_shutdown) +add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-graceful-shutdown) + add_subdirectory(https) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-https) diff --git a/core/functional_tests/basic_chaos/tests-restart/conftest.py b/core/functional_tests/basic_chaos/tests-restart/conftest.py index f8c9fff52d60..699ab947e59b 100644 --- a/core/functional_tests/basic_chaos/tests-restart/conftest.py +++ b/core/functional_tests/basic_chaos/tests-restart/conftest.py @@ -1 +1 @@ -pytest_plugins = ['pytest_userver.plugins.core', 'pytest_userver.plugins'] +pytest_plugins = ['pytest_userver.plugins.core'] diff --git a/core/functional_tests/basic_chaos/tests-restart/test_restart.py b/core/functional_tests/basic_chaos/tests-restart/test_restart.py index 9bde5cff2d64..c70546cdcc26 100644 --- a/core/functional_tests/basic_chaos/tests-restart/test_restart.py +++ b/core/functional_tests/basic_chaos/tests-restart/test_restart.py @@ -23,4 +23,4 @@ async def test_restart(monitor_client, service_client, _global_daemon_store): response = await service_client.get('/ping') assert response.status == 500 - wait_for_daemon_stop(_global_daemon_store) + await wait_for_daemon_stop(_global_daemon_store) diff --git a/core/functional_tests/graceful_shutdown/CMakeLists.txt b/core/functional_tests/graceful_shutdown/CMakeLists.txt new file mode 100644 index 000000000000..5e25e2bb10b8 --- /dev/null +++ b/core/functional_tests/graceful_shutdown/CMakeLists.txt @@ -0,0 +1,8 @@ +project(userver-core-tests-graceful-shutdown CXX) + +file(GLOB_RECURSE SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*pp") +add_executable(${PROJECT_NAME} ${SOURCES} "main.cpp") +target_include_directories(${PROJECT_NAME} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") +target_link_libraries(${PROJECT_NAME} userver-core) + +userver_chaos_testsuite_add() diff --git a/core/functional_tests/graceful_shutdown/main.cpp b/core/functional_tests/graceful_shutdown/main.cpp new file mode 100644 index 000000000000..f4d8df6e678c --- /dev/null +++ b/core/functional_tests/graceful_shutdown/main.cpp @@ -0,0 +1,12 @@ +#include +#include +#include +#include + +#include + +int main(int argc, char* argv[]) { + const auto component_list = + components::MinimalServerComponentList().Append().Append(); + return utils::DaemonMain(argc, argv, component_list); +} diff --git a/core/functional_tests/graceful_shutdown/src/handler_sigterm.hpp b/core/functional_tests/graceful_shutdown/src/handler_sigterm.hpp new file mode 100644 index 000000000000..eac6c4a8ee14 --- /dev/null +++ b/core/functional_tests/graceful_shutdown/src/handler_sigterm.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +#include +#include + +namespace handlers { + +class Sigterm final : public server::handlers::HttpHandlerBase { +public: + static constexpr std::string_view kName = "handler-sigterm"; + + Sigterm(const components::ComponentConfig& config, const components::ComponentContext& context) + : HttpHandlerBase(config, context) {} + + std::string HandleRequestThrow(const server::http::HttpRequest& /*request*/, server::request::RequestContext&) + const override { + kill(getpid(), SIGTERM); + return {}; + } +}; + +} // namespace handlers diff --git a/core/functional_tests/graceful_shutdown/static_config.yaml b/core/functional_tests/graceful_shutdown/static_config.yaml new file mode 100644 index 000000000000..b413c318faa7 --- /dev/null +++ b/core/functional_tests/graceful_shutdown/static_config.yaml @@ -0,0 +1,38 @@ +components_manager: + task_processors: + main-task-processor: + worker_threads: 4 + + fs-task-processor: + worker_threads: 2 + + default_task_processor: main-task-processor + + components: + logging: + fs-task-processor: fs-task-processor + loggers: + default: + file_path: '@stderr' + level: debug + overflow_behavior: discard + + server: + listener: + port: 8080 + task_processor: main-task-processor + listener-monitor: + port: 8081 + task_processor: main-task-processor + + handler-ping: + path: /ping + method: GET + task_processor: main-task-processor + throttling_enabled: false + + handler-sigterm: + monitor-handler: true + path: /sigterm + method: POST + task_processor: main-task-processor diff --git a/core/functional_tests/graceful_shutdown/tests/conftest.py b/core/functional_tests/graceful_shutdown/tests/conftest.py new file mode 100644 index 000000000000..108cd1ba7d64 --- /dev/null +++ b/core/functional_tests/graceful_shutdown/tests/conftest.py @@ -0,0 +1,13 @@ +import pytest + +pytest_plugins = ['pytest_userver.plugins.core'] + + +# Overriding a userver fixture +@pytest.fixture(name='userver_config_testsuite', scope='session') +def _userver_config_testsuite(userver_config_testsuite): + def patch_config(config, config_vars) -> None: + userver_config_testsuite(config, config_vars) + config['components_manager']['graceful_shutdown_interval'] = '3s' + + return patch_config diff --git a/core/functional_tests/graceful_shutdown/tests/test_graceful_shutdown.py b/core/functional_tests/graceful_shutdown/tests/test_graceful_shutdown.py new file mode 100644 index 000000000000..bcc2656c8a2e --- /dev/null +++ b/core/functional_tests/graceful_shutdown/tests/test_graceful_shutdown.py @@ -0,0 +1,37 @@ +import asyncio +import datetime + + +async def wait_for_daemon_stop(_global_daemon_store): + deadline = datetime.datetime.now() + datetime.timedelta(seconds=10) + while ( + datetime.datetime.now() < deadline + and _global_daemon_store.has_running_daemons() + ): + await asyncio.sleep(0.05) + + assert ( + not _global_daemon_store.has_running_daemons() + ), 'Daemon has not stopped' + await _global_daemon_store.aclose() + + +async def test_graceful_shutdown_timer( + service_client, monitor_client, _global_daemon_store, +): + response = await service_client.get('/ping') + assert response.status == 200 + + response = await monitor_client.post('/sigterm') + assert response.status == 200 + + response = await service_client.get('/ping') + assert response.status == 500 + + await asyncio.sleep(2) + # Check that the service is still alive. + response = await service_client.get('/ping') + assert response.status == 500 + + # After a couple more seconds, the service will start shutting down. + await wait_for_daemon_stop(_global_daemon_store) diff --git a/core/include/userver/components/component_context.hpp b/core/include/userver/components/component_context.hpp index 74a2ffa8ab0c..1de242880a2b 100644 --- a/core/include/userver/components/component_context.hpp +++ b/core/include/userver/components/component_context.hpp @@ -178,6 +178,8 @@ class ComponentContext final { void OnAllComponentsLoaded(); + void OnGracefulShutdownStarted(); + void OnAllComponentsAreStopping(); void ClearComponents(); diff --git a/core/src/components/component_context.cpp b/core/src/components/component_context.cpp index f56f2538d639..42c66ec26e63 100644 --- a/core/src/components/component_context.cpp +++ b/core/src/components/component_context.cpp @@ -28,6 +28,8 @@ RawComponentBase* ComponentContext::AddComponent(std::string_view name, const im void ComponentContext::OnAllComponentsLoaded() { impl_->OnAllComponentsLoaded(); } +void ComponentContext::OnGracefulShutdownStarted() { impl_->OnGracefulShutdownStarted(); } + void ComponentContext::OnAllComponentsAreStopping() { impl_->OnAllComponentsAreStopping(); } void ComponentContext::ClearComponents() { impl_->ClearComponents(); } diff --git a/core/src/components/component_context_impl.cpp b/core/src/components/component_context_impl.cpp index 5ba064f1199e..25801b55b8bc 100644 --- a/core/src/components/component_context_impl.cpp +++ b/core/src/components/component_context_impl.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -136,6 +138,16 @@ void ComponentContextImpl::OnAllComponentsLoaded() { ); } +void ComponentContextImpl::OnGracefulShutdownStarted() { + shutdown_started_ = true; + + const auto interval = manager_.GetConfig().graceful_shutdown_interval; + if (interval > std::chrono::milliseconds{0}) { + LOG_INFO() << "Shutdown started, notifying ping handlers and delaying by " << interval; + engine::SleepFor(interval); + } +} + void ComponentContextImpl::OnAllComponentsAreStopping() { LOG_INFO() << "Sending stopping notification to all components"; ProcessAllComponentLifetimeStageSwitchings( @@ -209,6 +221,11 @@ bool ComponentContextImpl::IsAnyComponentInFatalState() const { } } + if (shutdown_started_) { + LOG_WARNING() << "Service is shutting down, returning 5xx from ping"; + return true; + } + return false; } diff --git a/core/src/components/component_context_impl.hpp b/core/src/components/component_context_impl.hpp index 5f2a1c4f2e99..9cd883ba5eeb 100644 --- a/core/src/components/component_context_impl.hpp +++ b/core/src/components/component_context_impl.hpp @@ -31,6 +31,8 @@ class ComponentContextImpl { void OnAllComponentsLoaded(); + void OnGracefulShutdownStarted(); + void OnAllComponentsAreStopping(); void ClearComponents(); @@ -155,6 +157,8 @@ class ComponentContextImpl { engine::ConditionVariable print_adding_components_cv_; concurrent::Variable shared_data_; engine::TaskWithResult print_adding_components_task_; + + std::atomic shutdown_started_{false}; }; } // namespace components::impl diff --git a/core/src/components/manager.cpp b/core/src/components/manager.cpp index 64ba96a526f9..7cfe318a1612 100644 --- a/core/src/components/manager.cpp +++ b/core/src/components/manager.cpp @@ -191,6 +191,11 @@ Manager::Manager(std::unique_ptr&& config, const ComponentList& c Manager::~Manager() { LOG_INFO() << "Stopping components manager"; + try { + RunInCoro(*default_task_processor_, [this] { component_context_.OnGracefulShutdownStarted(); }); + } catch (const std::exception& exc) { + LOG_ERROR() << "Graceful shutdown failed: " << exc; + } engine::impl::TeardownPhdrCacheAndEnableDynamicLoading(); LOG_TRACE() << "Stopping component context"; diff --git a/core/src/components/manager_config.cpp b/core/src/components/manager_config.cpp index 9a0d813f3288..1dd7b6fdbff6 100644 --- a/core/src/components/manager_config.cpp +++ b/core/src/components/manager_config.cpp @@ -221,6 +221,13 @@ additionalProperties: false additionalProperties: type: boolean description: whether a specific experiment is enabled + graceful_shutdown_interval: + type: string + description: | + At shutdown, first hang for this duration with /ping 5xx to give + the balancer a chance to redirect new requests to other hosts and + to give the service a chance to finish handling old requests. + defaultDescription: 0s )"); } @@ -251,6 +258,9 @@ ManagerConfig Parse(const yaml_config::YamlConfig& value, formats::parse::To(config.disable_phdr_cache); config.preheat_stacktrace_collector = value["preheat_stacktrace_collector"].As(config.preheat_stacktrace_collector); + config.graceful_shutdown_interval = + value["graceful_shutdown_interval"].As(config.graceful_shutdown_interval); + return config; } diff --git a/core/src/components/manager_config.hpp b/core/src/components/manager_config.hpp index 417e077ca77d..33e4fd423876 100644 --- a/core/src/components/manager_config.hpp +++ b/core/src/components/manager_config.hpp @@ -25,6 +25,7 @@ struct ManagerConfig { std::string default_task_processor; ValidationMode validate_components_configs{}; utils::impl::UserverExperimentSet enabled_experiments; + std::chrono::milliseconds graceful_shutdown_interval{}; bool mlock_debug_info{true}; bool disable_phdr_cache{false}; bool preheat_stacktrace_collector{true}; diff --git a/testsuite/pytest_plugins/pytest_userver/plugins/config.py b/testsuite/pytest_plugins/pytest_userver/plugins/config.py index f39a4ce83617..cd272f176511 100644 --- a/testsuite/pytest_plugins/pytest_userver/plugins/config.py +++ b/testsuite/pytest_plugins/pytest_userver/plugins/config.py @@ -504,6 +504,8 @@ def _disable_cache_periodic_update(testsuite_support: dict) -> None: testsuite_support['testsuite-periodic-update-enabled'] = False def patch_config(config, config_vars) -> None: + # Don't delay tests teardown unnecessarily. + config['components_manager'].pop('graceful_shutdown_interval', None) components: dict = config['components_manager']['components'] if 'testsuite-support' not in components: return