Skip to content

Commit 11113b1

Browse files
Arun Yerrameta-codesync[bot]
Arun Yerra
authored and
meta-codesync[bot]
committed
Bechmark Test for ARS mode ECMP Group scaling.
Summary: Benchmark test with the following ECMP groups and ECMP group members. TH5 - 128 ECMP groups with 64 group members on Tomhawk5 ASIC. TH5 - 256 ECMP groups with 64 group members with enchanced ARS mode on Tomhawk5 ASIC. TH4 - 128 ECMP groups with 64 group members on Tomhawk4 ASIC. Differential Revision: D85838799 fbshipit-source-id: 96e866f44a4b4f776ab3bf37ad1c6f64c59715be
1 parent 4c34479 commit 11113b1

File tree

2 files changed

+184
-0
lines changed

2 files changed

+184
-0
lines changed

‎fboss/agent/hw/benchmarks/BUCK‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,19 @@ agent_benchmark_lib(
498498
],
499499
)
500500

501+
agent_benchmark_lib(
502+
name = "hw_ecmp_group_scale_benchmark",
503+
srcs = ["HwEcmpGroupScaleBenchmark.cpp"],
504+
extra_deps = [
505+
"//fboss/agent/hw/test:hw_switch_ensemble_factory",
506+
"//fboss/agent/hw/test:hw_test_ecmp_utils",
507+
"//fboss/agent/hw/test:hw_test_port_utils",
508+
"//fboss/agent/test/utils:ecmp_test_utils",
509+
"//fboss/agent/test/utils:load_balancer_test_utils",
510+
"//fboss/agent/test/utils:scale_test_utils",
511+
],
512+
)
513+
501514
cpp_library(
502515
name = "hw_ucmp_group_scale_helper",
503516
headers = ["HwUcmpScaleBenchmarkHelper.h"],

‎fboss/agent/hw/benchmarks/HwEcmpGroupScaleBenchmark.cpp‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
/*
2+
* Copyright (c) 2004-present, Facebook, Inc.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree. An additional grant
7+
* of patent rights can be found in the PATENTS file in the same directory.
8+
*
9+
*/
10+
11+
#include "fboss/agent/AddressUtil.h"
12+
#include "fboss/agent/hw/test/HwTestEcmpUtils.h"
13+
#include "fboss/agent/test/AgentEnsemble.h"
14+
#include "fboss/agent/test/EcmpSetupHelper.h"
15+
#include "fboss/agent/test/utils/ConfigUtils.h"
16+
#include "fboss/agent/test/utils/LoadBalancerTestUtils.h"
17+
#include "fboss/agent/test/utils/ScaleTestUtils.h"
18+
19+
#include <folly/Benchmark.h>
20+
#include <folly/IPAddress.h>
21+
22+
#include "fboss/agent/SwSwitchRouteUpdateWrapper.h"
23+
24+
namespace facebook::fboss {
25+
26+
void unprogramRoutes(
27+
RouteUpdateWrapper* updater,
28+
const std::vector<RoutePrefixV6>& prefixes) {
29+
for (const auto& prefix : prefixes) {
30+
updater->delRoute(
31+
RouterID(0),
32+
folly::IPAddress(prefix.network()),
33+
prefix.mask(),
34+
ClientID::BGPD);
35+
}
36+
}
37+
38+
void programRoutes(
39+
RouteUpdateWrapper* updater,
40+
const std::vector<std::vector<PortDescriptor>>& portDescs,
41+
const std::vector<RoutePrefixV6>& prefixes,
42+
const utility::EcmpSetupTargetedPorts<folly::IPAddressV6>* ecmpHelper) {
43+
auto constexpr kClientID(ClientID::BGPD);
44+
const AdminDistance kDefaultAdminDistance = AdminDistance::EBGP;
45+
for (auto i = 0; i < prefixes.size(); ++i) {
46+
RouteNextHopSet nhopSet;
47+
for (const auto& portDesc : portDescs[i]) {
48+
auto nhop = ecmpHelper->nhop(portDesc);
49+
nhopSet.emplace(ResolvedNextHop(nhop.ip, nhop.intf, UCMP_DEFAULT_WEIGHT));
50+
}
51+
RouteNextHopEntry nhopEntry(nhopSet, kDefaultAdminDistance);
52+
updater->addRoute(
53+
RouterID(0),
54+
prefixes[i].network(),
55+
prefixes[i].mask(),
56+
kClientID,
57+
nhopEntry);
58+
}
59+
}
60+
61+
void ecmpGroupScaleBenchmark(
62+
bool add,
63+
uint32_t ecmpGroups,
64+
uint32_t ecmpGroupWidth) {
65+
folly::BenchmarkSuspender suspender;
66+
AgentEnsembleSwitchConfigFn initialConfigFn =
67+
[&](const AgentEnsemble& ensemble) {
68+
FLAGS_enable_ecmp_resource_manager = true;
69+
FLAGS_ecmp_resource_percentage = 100;
70+
FLAGS_flowletSwitchingEnable = true;
71+
FLAGS_dlbResourceCheckEnable = false;
72+
FLAGS_enable_th5_ars_scale_mode = false;
73+
auto cfg = utility::onePortPerInterfaceConfig(
74+
ensemble.getSw(), ensemble.masterLogicalPortIds());
75+
76+
// If the switch ASIC is TH5 and max desired ECMP group members is
77+
// 256, enable TH5 ARS scale mode
78+
auto switchIds = ensemble.getSw()->getHwAsicTable()->getSwitchIDs();
79+
CHECK_GE(switchIds.size(), 1);
80+
auto asicType = ensemble.getSw()
81+
->getHwAsicTable()
82+
->getHwAsic(*switchIds.cbegin())
83+
->getAsicType();
84+
/* if (ecmpGroups == 256) {
85+
CHECK_EQ(asicType, cfg::AsicType::ASIC_TYPE_TOMAHAWK5);
86+
} */
87+
if ((asicType == cfg::AsicType::ASIC_TYPE_TOMAHAWK5) &&
88+
(ecmpGroups == 256)) {
89+
FLAGS_enable_th5_ars_scale_mode = true;
90+
}
91+
XLOG(DBG2) << "EcmpGroupWidth: " << ecmpGroupWidth
92+
<< ", EcmpGroups: " << ecmpGroups << ", th5_ars_scale_mode: "
93+
<< FLAGS_enable_th5_ars_scale_mode;
94+
95+
utility::addFlowletConfigs(
96+
cfg,
97+
ensemble.masterLogicalPortIds(),
98+
ensemble.isSai(),
99+
cfg::SwitchingMode::PER_PACKET_QUALITY);
100+
return cfg;
101+
};
102+
auto ensemble =
103+
createAgentEnsemble(initialConfigFn, false /*disableLinkStateToggler*/);
104+
auto ecmpHelper = utility::EcmpSetupTargetedPorts6(
105+
ensemble->getSw()->getState(),
106+
ensemble->getSw()->needL2EntryForNeighbor());
107+
boost::container::flat_set<PortDescriptor> portDescs;
108+
109+
for (const auto& portId : ensemble->masterLogicalInterfacePortIds()) {
110+
portDescs.insert(PortDescriptor(portId));
111+
}
112+
ensemble->applyNewState([&](const std::shared_ptr<SwitchState>& in) {
113+
return ecmpHelper.resolveNextHops(in, portDescs);
114+
});
115+
116+
auto nhopSets = utility::generateEcmpGroupScale(
117+
std::vector<PortDescriptor>(portDescs.begin(), portDescs.end()),
118+
ecmpGroups,
119+
ecmpGroupWidth /*max ecmp width*/,
120+
ecmpGroupWidth /*min ecmp width*/);
121+
std::vector<RoutePrefixV6> prefixes;
122+
prefixes.reserve(ecmpGroups);
123+
for (auto i = 0; i < ecmpGroups; ++i) {
124+
prefixes.emplace_back(
125+
folly::IPAddressV6(folly::to<std::string>(2401, ":", i + 1, "::")), 64);
126+
}
127+
128+
auto updater = ensemble->getSw()->getRouteUpdater();
129+
XLOG(DBG2) << "Operation: " << add << ", EcmpGroupWidth: " << ecmpGroupWidth
130+
<< ", EcmpGroups: " << ecmpGroups
131+
<< ", th5_ars_scale_mode: " << FLAGS_enable_th5_ars_scale_mode;
132+
133+
programRoutes(&updater, nhopSets, prefixes, &ecmpHelper);
134+
if (add) {
135+
suspender.dismiss();
136+
updater.program();
137+
suspender.rehire();
138+
unprogramRoutes(&updater, prefixes);
139+
updater.program();
140+
} else {
141+
updater.program();
142+
suspender.dismiss();
143+
unprogramRoutes(&updater, prefixes);
144+
updater.program();
145+
suspender.rehire();
146+
}
147+
}
148+
149+
BENCHMARK(RouteAddHwEcmpGroupScale128x64Benchmark) {
150+
// Measure 128x64 ECMP route add
151+
ecmpGroupScaleBenchmark(
152+
true /* add */, 128 /* ecmpGroup */, 64 /* ecmpWidth */);
153+
}
154+
BENCHMARK(RouteAddHwEcmpGroupScale256x64Benchmark) {
155+
// Measure 256x64 ECMP route add
156+
ecmpGroupScaleBenchmark(
157+
true /* add */, 256 /* ecmpGroup */, 64 /* ecmpWidth */);
158+
}
159+
160+
BENCHMARK(RouteDelHwEcmpGroupScale128x64Benchmark) {
161+
// Measure 128x64 ECMP route del
162+
ecmpGroupScaleBenchmark(
163+
false /* delete */, 128 /* ecmpGroup */, 64 /* ecmpWidth */);
164+
}
165+
BENCHMARK(RouteDelHwEcmpGroupScale256x64Benchmark) {
166+
// Measure 256x64 ECMP route del
167+
ecmpGroupScaleBenchmark(
168+
false /* delete */, 256 /* ecmpGroup */, 64 /* ecmpWidth */);
169+
}
170+
171+
} // namespace facebook::fboss

0 commit comments

Comments
 (0)