diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index b5c5ce51fcbc..dadd19daa93c 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -1125,6 +1125,8 @@ message ScopedRoutes { // in this message. ScopedRds scoped_rds = 5; } + + google.protobuf.BoolValue retry_other_scope_when_not_found = 101; } message ScopedRds { diff --git a/envoy/router/scopes.h b/envoy/router/scopes.h index 8e4dbb7a646b..fa9cf737ffc5 100644 --- a/envoy/router/scopes.h +++ b/envoy/router/scopes.h @@ -123,11 +123,15 @@ class ScopedConfig : public Envoy::Config::ConfigProvider::Config { virtual ConfigConstSharedPtr getRouteConfig(const ScopeKeyPtr& scope_key) const PURE; #if defined(HIGRESS) - virtual ConfigConstSharedPtr - getRouteConfig(const ScopeKeyBuilder* builder, const Http::HeaderMap& headers, - const StreamInfo::StreamInfo* info = nullptr) const PURE; + virtual ConfigConstSharedPtr getRouteConfig(const ScopeKeyBuilder* builder, + const Http::HeaderMap& headers, + const StreamInfo::StreamInfo* info) const PURE; + virtual ConfigConstSharedPtr getRouteConfig(const ScopeKeyBuilder* builder, + const Http::HeaderMap& headers, + const StreamInfo::StreamInfo* info, + std::function& recompute) const PURE; virtual ScopeKeyPtr computeScopeKey(const ScopeKeyBuilder*, const Http::HeaderMap&, - const StreamInfo::StreamInfo* = nullptr) const { + const StreamInfo::StreamInfo*) const { return {}; }; #endif diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index ce2910e4182b..f3652f7c64aa 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -545,6 +545,11 @@ class ConnectionManagerConfig { * Zero indicates this behavior is disabled. */ virtual std::chrono::seconds keepaliveHeaderTimeout() const PURE; + /** + * @return whether to retry to other scoped routes when the target route is not found in the + * current scope, supported only when using scoped_routes. + */ + virtual bool retryOtherScopeWhenNotFound() const PURE; #endif }; } // namespace Http diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 9830cf95c49c..fd4faff0e32f 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1209,7 +1209,14 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt connection_manager_.config_.scopeKeyBuilder().has_value()) { snapped_scoped_routes_config_ = connection_manager_.config_.scopedRouteConfigProvider()->config(); +#if defined(HIGRESS) + // It is only used to determine whether to remove specific internal headers, but at the cost + // of an additional routing calculation. In our scenario, there is no removal of internal + // headers, so there is no need to calculate the route here. + snapped_route_config_ = std::make_shared(); +#else snapScopedRouteConfig(); +#endif } } else { snapped_route_config_ = connection_manager_.config_.routeConfigProvider()->configCast(); @@ -1524,9 +1531,10 @@ void ConnectionManagerImpl::startDrainSequence() { void ConnectionManagerImpl::ActiveStream::snapScopedRouteConfig() { #if defined(HIGRESS) + snapped_scoped_routes_recompute_ = nullptr; snapped_route_config_ = snapped_scoped_routes_config_->getRouteConfig( connection_manager_.config_.scopeKeyBuilder().ptr(), *request_headers_, - &connection()->streamInfo()); + &connection()->streamInfo(), snapped_scoped_routes_recompute_); #else // NOTE: if a RDS subscription hasn't got a RouteConfiguration back, a Router::NullConfigImpl is // returned, in that case we let it pass. @@ -1653,6 +1661,25 @@ void ConnectionManagerImpl::ActiveStream::refreshCachedRoute(const Router::Route } } +#if defined(HIGRESS) + if (connection_manager_.config_.retryOtherScopeWhenNotFound()) { + while (route == nullptr && snapped_scoped_routes_recompute_ != nullptr) { + ASSERT(snapped_scoped_routes_config_ != nullptr); + snapped_route_config_ = snapped_scoped_routes_config_->getRouteConfig( + connection_manager_.config_.scopeKeyBuilder().ptr(), *request_headers_, + &connection()->streamInfo(), snapped_scoped_routes_recompute_); + if (snapped_route_config_ == nullptr) { + break; + } + route = snapped_route_config_->route(cb, *request_headers_, filter_manager_.streamInfo(), + stream_id_); + ENVOY_STREAM_LOG(debug, + "after the route was not found, search again in other scopes and found:{}", + *this, route != nullptr); + } + } +#endif + setRoute(route); } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 859fd18a546f..f18177a21824 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -486,6 +486,9 @@ class ConnectionManagerImpl : Logger::Loggable, // route configuration is updated frequently and the request is long-lived. Router::ConfigConstSharedPtr snapped_route_config_; Router::ScopedConfigConstSharedPtr snapped_scoped_routes_config_; +#if defined(HIGRESS) + std::function snapped_scoped_routes_recompute_; +#endif // This is used to track the route that has been cached in the request. And we will keep this // route alive until the request is finished. absl::optional cached_route_; diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index bd4a6412a318..0ad3aabff5ef 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -91,9 +91,10 @@ HostValueExtractorImpl::computeFragment(const Http::HeaderMap& headers, if (port_start != absl::string_view::npos) { host = host.substr(0, port_start); } - *recompute = [this, host, weak_recompute = ReComputeCbWeakPtr(recompute)]() mutable - -> std::unique_ptr { - return reComputeHelper(std::string(host), weak_recompute, 0); + *recompute = [this, host_str = std::string(host), + weak_recompute = ReComputeCbWeakPtr( + recompute)]() mutable -> std::unique_ptr { + return reComputeHelper(host_str, weak_recompute, 0); }; return std::make_unique(host); } @@ -136,13 +137,14 @@ ScopeKeyPtr ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers, recompute = [&recompute, recompute_cbs]() mutable -> ScopeKeyPtr { ScopeKey new_key; for (auto& cb : *recompute_cbs) { + if (*cb == nullptr) { + recompute = nullptr; + return nullptr; + } auto new_fragment = (*cb)(); if (new_fragment == nullptr) { return nullptr; } - if (*cb == nullptr) { - recompute = nullptr; - } new_key.addFragment(std::move(new_fragment)); } return std::make_unique(std::move(new_key)); @@ -171,10 +173,14 @@ ScopeKeyPtr ScopedConfigImpl::computeScopeKey(const ScopeKeyBuilder* scope_key_b Router::ConfigConstSharedPtr ScopedConfigImpl::getRouteConfig(const ScopeKeyBuilder* scope_key_builder, - const Http::HeaderMap& headers, - const StreamInfo::StreamInfo* info) const { - std::function recompute; - ScopeKeyPtr scope_key = scope_key_builder->computeScopeKey(headers, info, recompute); + const Http::HeaderMap& headers, const StreamInfo::StreamInfo* info, + std::function& recompute) const { + ScopeKeyPtr scope_key = nullptr; + if (recompute == nullptr) { + scope_key = scope_key_builder->computeScopeKey(headers, info, recompute); + } else { + scope_key = recompute(); + } if (scope_key == nullptr) { return nullptr; } @@ -189,6 +195,14 @@ ScopedConfigImpl::getRouteConfig(const ScopeKeyBuilder* scope_key_builder, return nullptr; } +Router::ConfigConstSharedPtr +ScopedConfigImpl::getRouteConfig(const ScopeKeyBuilder* scope_key_builder, + const Http::HeaderMap& headers, + const StreamInfo::StreamInfo* info) const { + std::function recompute; + return getRouteConfig(scope_key_builder, headers, info, recompute); +} + #endif bool ScopeKey::operator!=(const ScopeKey& other) const { return !(*this == other); } diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 96425a4c74a2..a71de917a91b 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -178,6 +178,10 @@ class ScopedConfigImpl : public ScopedConfig { Router::ConfigConstSharedPtr getRouteConfig(const ScopeKeyPtr& scope_key) const override; #if defined(HIGRESS) + Router::ConfigConstSharedPtr + getRouteConfig(const ScopeKeyBuilder* scope_key_builder, const Http::HeaderMap& headers, + const StreamInfo::StreamInfo* info, + std::function& recompute) const override; Router::ConfigConstSharedPtr getRouteConfig(const ScopeKeyBuilder* scope_key_builder, const Http::HeaderMap& headers, const StreamInfo::StreamInfo* info) const override; @@ -202,6 +206,11 @@ class NullScopedConfigImpl : public ScopedConfig { return std::make_shared(); } #if defined(HIGRESS) + Router::ConfigConstSharedPtr getRouteConfig(const ScopeKeyBuilder*, const Http::HeaderMap&, + const StreamInfo::StreamInfo*, + std::function&) const override { + return std::make_shared(); + } Router::ConfigConstSharedPtr getRouteConfig(const ScopeKeyBuilder*, const Http::HeaderMap&, const StreamInfo::StreamInfo*) const override { return std::make_shared(); diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 1d6ce67d75c6..640f00d32596 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -498,6 +498,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( config, context_.getServerFactoryContext(), context_.initManager(), stats_prefix_, scoped_routes_config_provider_manager_); scope_key_builder_ = Router::ScopedRoutesConfigProviderUtil::createScopeKeyBuilder(config); + retry_other_scope_when_not_found_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT( + config.scoped_routes(), retry_other_scope_when_not_found, true); break; case envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager:: RouteSpecifierCase::ROUTE_SPECIFIER_NOT_SET: diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index 39c77861d6dd..c105d807623a 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -269,6 +269,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, } #if defined(HIGRESS) std::chrono::seconds keepaliveHeaderTimeout() const override { return keepalive_header_timeout_; } + bool retryOtherScopeWhenNotFound() const override { return retry_other_scope_when_not_found_; } #endif private: @@ -332,6 +333,9 @@ class HttpConnectionManagerConfig : Logger::Loggable, // routes Router::ScopeKeyBuilderPtr scope_key_builder_; Config::ConfigProviderPtr scoped_routes_config_provider_; +#if defined(HIGRESS) + bool retry_other_scope_when_not_found_; +#endif std::chrono::milliseconds drain_timeout_; bool generate_request_id_; const bool preserve_external_request_id_; diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 6f4233d575bc..77d1778d2c52 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -229,6 +229,7 @@ class AdminImpl : public Admin, bool addProxyProtocolConnectionState() const override { return true; } #if defined(HIGRESS) std::chrono::seconds keepaliveHeaderTimeout() const override { return {}; } + bool retryOtherScopeWhenNotFound() const override { return false; } #endif private: diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 0ec5bf485599..d7ae59ec6877 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -243,6 +243,7 @@ class FuzzConfig : public ConnectionManagerConfig { bool addProxyProtocolConnectionState() const override { return true; } #if defined(HIGRESS) std::chrono::seconds keepaliveHeaderTimeout() const override { return keepalive_header_timeout_; } + bool retryOtherScopeWhenNotFound() const override { return retry_other_scope_when_not_found_; } #endif const envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager @@ -299,6 +300,7 @@ class FuzzConfig : public ConnectionManagerConfig { std::unique_ptr proxy_status_config_; #if defined(HIGRESS) std::chrono::seconds keepalive_header_timeout_{}; + bool retry_other_scope_when_not_found_ = false; #endif }; diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index 93c71e44bb2d..910164d4f2a1 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -2753,7 +2753,7 @@ TEST_F(HttpConnectionManagerImplTest, TestSrdsRouteNotFound) { #if defined(HIGRESS) EXPECT_CALL(*static_cast( scopedRouteConfigProvider()->config().get()), - getRouteConfig(_, _, _)) + getRouteConfig(_, _, _, _)) .Times(2) .WillRepeatedly(Return(nullptr)); #else @@ -2796,7 +2796,7 @@ TEST_F(HttpConnectionManagerImplTest, TestSrdsUpdate) { #if defined(HIGRESS) EXPECT_CALL(*static_cast( scopedRouteConfigProvider()->config().get()), - getRouteConfig(_, _, _)) + getRouteConfig(_, _, _, _)) .Times(3) .WillOnce(Return(nullptr)) .WillOnce(Return(nullptr)) // refreshCachedRoute first time. @@ -2870,19 +2870,21 @@ TEST_F(HttpConnectionManagerImplTest, TestSrdsCrossScopeReroute) { #if defined(HIGRESS) EXPECT_CALL(*static_cast( scopedRouteConfigProvider()->config().get()), - getRouteConfig(_, _, _)) + getRouteConfig(_, _, _, _)) // 1. Snap scoped route config; // 2. refreshCachedRoute (both in decodeHeaders(headers,end_stream); // 3. then refreshCachedRoute triggered by decoder_filters_[1]->callbacks_->route(). .Times(3) - .WillRepeatedly(Invoke([&](const Router::ScopeKeyBuilder*, const Http::HeaderMap& headers, - const StreamInfo::StreamInfo*) -> Router::ConfigConstSharedPtr { - auto& test_headers = dynamic_cast(headers); - if (test_headers.get_("scope_key") == "foo") { - return route_config1; - } - return route_config2; - })); + .WillRepeatedly( + Invoke([&](const Router::ScopeKeyBuilder*, const Http::HeaderMap& headers, + const StreamInfo::StreamInfo*, + std::function&) -> Router::ConfigConstSharedPtr { + auto& test_headers = dynamic_cast(headers); + if (test_headers.get_("scope_key") == "foo") { + return route_config1; + } + return route_config2; + })); #else EXPECT_CALL(*static_cast(scopeKeyBuilder().ptr()), computeScopeKey(_)) @@ -2959,7 +2961,7 @@ TEST_F(HttpConnectionManagerImplTest, TestSrdsRouteFound) { EXPECT_CALL(cluster_manager_, getThreadLocalCluster(_)).WillOnce(Return(fake_cluster1.get())); #if defined(HIGRESS) EXPECT_CALL(*scopedRouteConfigProvider()->config(), - getRouteConfig(_, _, _)) + getRouteConfig(_, _, _, _)) // 1. decodeHeaders() snapping route config. // 2. refreshCachedRoute() later in the same decodeHeaders(). .Times(2); diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index 5472fb7a8139..44f5b20f2a05 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -178,6 +178,7 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { #if defined(HIGRESS) std::chrono::seconds keepaliveHeaderTimeout() const override { return keepalive_header_timeout_; } + bool retryOtherScopeWhenNotFound() const override { return retry_other_scope_when_not_found_; } #endif // Simple helper to wrapper filter to the factory function. FilterFactoryCb createDecoderFilterFactoryCb(StreamDecoderFilterSharedPtr filter) { @@ -282,6 +283,7 @@ class HttpConnectionManagerImplMixin : public ConnectionManagerConfig { bool add_proxy_protocol_connection_state_ = true; #if defined(HIGRESS) std::chrono::seconds keepalive_header_timeout_{}; + bool retry_other_scope_when_not_found_ = true; #endif const LocalReply::LocalReplyPtr local_reply_; diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 510f59965d17..fa16e2d52365 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -253,6 +253,77 @@ TEST_F(InlineScopedRoutesTest, InlineRouteConfigurations) { "foo"); } +#if defined(HIGRESS) +TEST_F(InlineScopedRoutesTest, InlineWildcardDomainFallback) { + server_factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); + const std::string hcm_config = absl::StrCat(hcm_config_base, R"EOF( +scoped_routes: + name: $0 + scope_key_builder: + fragments: + - local_port_value_extractor: {} + - host_value_extractor: {} + scoped_route_configurations_list: + scoped_route_configurations: + - name: foo-scope + route_configuration: + name: foo + virtual_hosts: + - name: bar + domains: ["www.example.com"] + routes: + - match: { path: "/" } + route: { cluster: baz } + key: + fragments: + - string_key: "80" + - string_key: www.example.com + - name: foo2-scope + route_configuration: + name: foo2 + virtual_hosts: + - name: bar + domains: ["*.example.com"] + routes: + - match: { path: "/foo" } + route: { cluster: baz } + key: + fragments: + - string_key: "80" + - string_key: "*.example.com" +)EOF"); + const auto config = + parseHttpConnectionManagerFromYaml(absl::Substitute(hcm_config, "foo-scoped-routes")); + Envoy::Config::ConfigProviderPtr provider = ScopedRoutesConfigProviderUtil::create( + config, server_factory_context_, context_init_manager_, "foo.", *config_provider_manager_); + Envoy::Router::ScopeKeyBuilderPtr scope_key_builder = + ScopedRoutesConfigProviderUtil::createScopeKeyBuilder(config); + ASSERT_THAT(provider->config(), Not(IsNull())); + + NiceMock stream_info; + auto downstream_connection_info_provider = std::make_shared( + std::make_shared("127.0.0.1", 80), + std::make_shared("127.0.0.2", 1000)); + ON_CALL(stream_info, downstreamAddressProvider()) + .WillByDefault(ReturnPointee(downstream_connection_info_provider)); + + std::function recompute; + Http::TestRequestHeaderMapImpl headers = {{":authority", "www.example.com"}, + {":path", "/foo"}, + {":method", "GET"}, + {":scheme", "http"}, + {"x-forwarded-proto", "http"}}; + auto route_config = provider->config()->getRouteConfig( + scope_key_builder.get(), headers, &stream_info, recompute); + EXPECT_EQ(route_config->name(), "foo"); + EXPECT_EQ(route_config->route(headers, stream_info, 0), nullptr); + route_config = provider->config()->getRouteConfig( + scope_key_builder.get(), headers, &stream_info, recompute); + EXPECT_EQ(route_config->name(), "foo2"); + EXPECT_NE(route_config->route(headers, stream_info, 0), nullptr); +} +#endif + TEST_F(InlineScopedRoutesTest, ConfigLoadAndDump) { server_factory_context_.cluster_manager_.initializeClusters({"baz"}, {}); timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index b52be457f564..41743445b133 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -680,6 +680,7 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_METHOD(bool, addProxyProtocolConnectionState, (), (const)); #if defined(HIGRESS) MOCK_METHOD(std::chrono::seconds, keepaliveHeaderTimeout, (), (const)); + MOCK_METHOD(bool, retryOtherScopeWhenNotFound, (), (const)); #endif std::unique_ptr internal_address_config_ = diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index 72a6f6cb1381..a89cba4e458c 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -175,6 +175,7 @@ MockScopedConfig::MockScopedConfig() { ON_CALL(*this, getRouteConfig(_)).WillByDefault(Return(route_config_)); #if defined(HIGRESS) ON_CALL(*this, getRouteConfig(_, _, _)).WillByDefault(Return(route_config_)); + ON_CALL(*this, getRouteConfig(_, _, _, _)).WillByDefault(Return(route_config_)); #endif } MockScopedConfig::~MockScopedConfig() = default; diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 84807f146e45..de385fe27408 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -619,6 +619,10 @@ class MockScopedConfig : public ScopedConfig { MOCK_METHOD(ConfigConstSharedPtr, getRouteConfig, (const ScopeKeyBuilder*, const Http::HeaderMap&, const StreamInfo::StreamInfo*), (const)); + MOCK_METHOD(ConfigConstSharedPtr, getRouteConfig, + (const ScopeKeyBuilder*, const Http::HeaderMap&, const StreamInfo::StreamInfo*, + std::function&), + (const)); #endif std::shared_ptr route_config_{new NiceMock()};