diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh index 6525bd5e54ac16..7b81d92e9af9ef 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh @@ -113,7 +113,7 @@ buildConfigs() { -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=scalable \ + -a pool="${pool}" \ --allow_client_language=c++ --allow_server_language=c++ \ --allow_server_language=node \ -o "loadtest_with_prebuilt_workers_${pool}.yaml" @@ -139,8 +139,8 @@ for language in "${disabledLanguages[@]}"; do done # Add arguments for languages. -declare -a configLangArgs8core=() -declare -a configLangArgs32core=() +declare -a configLangArgs8core=("--category=scalable_limited") +declare -a configLangArgs32core=("--category=scalable") declare -a runnerLangArgs=() # c++ diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh index f15a716028a83f..fa2ae4ad2cf7dd 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh @@ -108,7 +108,7 @@ buildConfigs() { -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=scalable \ + -a pool="${pool}" \ --allow_client_language=c++ --allow_server_language=c++ \ --allow_server_language=node \ -o "loadtest_with_prebuilt_workers_${pool}.yaml" @@ -134,8 +134,8 @@ for language in "${disabledLanguages[@]}"; do done # Add arguments for languages. -declare -a configLangArgs8core=() -declare -a configLangArgs32core=() +declare -a configLangArgs8core=("--category=scalable_limited") +declare -a configLangArgs32core=("--category=scalable") declare -a runnerLangArgs=() # c++ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 0445f6bb9205c9..f2ef7ab39c60fa 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -23,13 +23,17 @@ SMOKETEST = "smoketest" SCALABLE = "scalable" +# The category below is for C++ benchmarks that can run on 8 cores. +# TODO research why this particular benchmark hangs on 8 cores but not on 30: +# cpp-protobuf-async-client-unary-1channel-64wide-128breq-8mbresp-insecure. +SCALABLE_LIMITED = "scalable_limited" INPROC = "inproc" SWEEP = "sweep" PSM = "psm" # A small superset of the benchmarks required to produce # https://grafana-dot-grpc-testing.appspot.com/ DASHBOARD = "dashboard" -DEFAULT_CATEGORIES = (SCALABLE, SMOKETEST) +DEFAULT_CATEGORIES = (SCALABLE, SCALABLE_LIMITED, SMOKETEST) SECURE_SECARGS = { "use_test_ca": True, @@ -321,7 +325,7 @@ def scenarios(self): ) # Scenario was added in https://github.com/grpc/grpc/pull/12987, but its purpose is unclear - # (beyond excercising some params that other scenarios don't) + # (beyond exercising some params that other scenarios don't) yield _ping_pong_scenario( "cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp", rpc_type="UNARY", @@ -336,7 +340,7 @@ def scenarios(self): secure=False, async_server_threads=16, server_threads_per_cq=1, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -355,7 +359,7 @@ def scenarios(self): secure=secure, categories=smoketest_categories + inproc_categories - + [SCALABLE], + + [SCALABLE, SCALABLE_LIMITED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -372,7 +376,7 @@ def scenarios(self): minimal_stack=not secure, categories=smoketest_categories + inproc_categories - + [SCALABLE], + + [SCALABLE, SCALABLE_LIMITED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -390,7 +394,7 @@ def scenarios(self): minimal_stack=not secure, categories=smoketest_categories + inproc_categories - + [SCALABLE], + + [SCALABLE, SCALABLE_LIMITED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -421,7 +425,7 @@ def scenarios(self): use_generic_payload=True, secure=secure, minimal_stack=not secure, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + [SCALABLE, SCALABLE_LIMITED], channels=1, outstanding=100, warmup_seconds=CXX_WARMUP_SECONDS, @@ -439,7 +443,7 @@ def scenarios(self): use_generic_payload=True, secure=secure, minimal_stack=not secure, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + [SCALABLE, SCALABLE_LIMITED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -467,7 +471,7 @@ def scenarios(self): secure=secure, client_threads_per_cq=1000000, server_threads_per_cq=1000000, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + [SCALABLE, SCALABLE_LIMITED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -480,7 +484,7 @@ def scenarios(self): secure=secure, client_threads_per_cq=1000000, server_threads_per_cq=1000000, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + [SCALABLE, SCALABLE_LIMITED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -509,10 +513,11 @@ def scenarios(self): minimal_stack=not secure, categories=smoketest_categories + inproc_categories - + [SCALABLE], + + [SCALABLE, SCALABLE_LIMITED], warmup_seconds=CXX_WARMUP_SECONDS, ) + # Insecure test hangs on 8 cores: excluded from SCALABLE_LIMITED. yield _ping_pong_scenario( "cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s" % (secstr), @@ -525,7 +530,8 @@ def scenarios(self): resp_size=8 * 1024 * 1024, secure=secure, minimal_stack=not secure, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + + ([SCALABLE, SCALABLE_LIMITED] if secure else [SCALABLE]), warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -553,7 +559,7 @@ def scenarios(self): minimal_stack=not secure, categories=smoketest_categories + inproc_categories - + [SCALABLE, DASHBOARD], + + [SCALABLE, SCALABLE_LIMITED, DASHBOARD], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -604,7 +610,7 @@ def scenarios(self): warmup_seconds=CXX_WARMUP_SECONDS, ) - maybe_scalable = [SCALABLE] + maybe_scalable = [SCALABLE, SCALABLE_LIMITED] if ( rpc_type == "streaming_from_server" and synchronicity == "async" @@ -647,7 +653,7 @@ def scenarios(self): # server_type='%s_SERVER' % synchronicity.upper(), # unconstrained_client=synchronicity, # secure=secure, - # categories=smoketest_categories+[SCALABLE], + # categories=smoketest_categories+[SCALABLE, SCALABLE_LIMITED], # warmup_seconds=CXX_WARMUP_SECONDS, # resource_quota_size=500*1024) @@ -663,7 +669,8 @@ def scenarios(self): secure=secure, messages_per_stream=mps, minimal_stack=not secure, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + + [SCALABLE, SCALABLE_LIMITED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -733,7 +740,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_GENERIC_SERVER", use_generic_payload=True, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -745,7 +752,7 @@ def scenarios(self): resp_size=1024 * 1024, use_generic_payload=True, secure=False, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -756,7 +763,7 @@ def scenarios(self): unconstrained_client="async", use_generic_payload=True, secure=False, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -771,7 +778,7 @@ def scenarios(self): rpc_type="UNARY", client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -787,7 +794,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", unconstrained_client="async", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -796,7 +803,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", unconstrained_client="async", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -806,7 +813,7 @@ def scenarios(self): server_type="SYNC_SERVER", server_language="c++", async_server_threads=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -825,7 +832,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", server_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -835,7 +842,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="sync", server_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -845,7 +852,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", client_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -855,7 +862,7 @@ def scenarios(self): server_type="ASYNC_SERVER", req_size=1024 * 1024, resp_size=1024 * 1024, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -880,7 +887,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_GENERIC_SERVER", use_generic_payload=True, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -892,7 +899,7 @@ def scenarios(self): resp_size=1024 * 1024, use_generic_payload=True, secure=False, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -903,7 +910,7 @@ def scenarios(self): unconstrained_client="async", use_generic_payload=True, secure=False, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -918,7 +925,7 @@ def scenarios(self): rpc_type="UNARY", client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -934,7 +941,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", unconstrained_client="async", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -943,7 +950,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", unconstrained_client="async", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -953,7 +960,7 @@ def scenarios(self): server_type="SYNC_SERVER", server_language="c++", async_server_threads=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -972,7 +979,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", server_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -982,7 +989,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="sync", server_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -992,7 +999,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", client_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -1002,7 +1009,7 @@ def scenarios(self): server_type="ASYNC_SERVER", req_size=1024 * 1024, resp_size=1024 * 1024, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -1038,7 +1045,7 @@ def scenarios(self): client_type="SYNC_CLIENT", server_type="ASYNC_GENERIC_SERVER", use_generic_payload=True, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1060,7 +1067,7 @@ def scenarios(self): rpc_type="UNARY", client_type="SYNC_CLIENT", server_type="ASYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1086,7 +1093,7 @@ def scenarios(self): server_type="ASYNC_SERVER", server_language="c++", async_server_threads=0, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1105,7 +1112,7 @@ def scenarios(self): server_type="ASYNC_SERVER", req_size=1024 * 1024, resp_size=1024 * 1024, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -1151,7 +1158,7 @@ def scenarios(self): client_processes=0, server_processes=0, unconstrained_client="async", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -1165,7 +1172,7 @@ def scenarios(self): client_processes=1, server_processes=1, unconstrained_client="async", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -1177,7 +1184,7 @@ def scenarios(self): client_processes=1, server_processes=1, use_generic_payload=True, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1188,7 +1195,7 @@ def scenarios(self): channels=1, client_processes=1, server_processes=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1198,7 +1205,7 @@ def scenarios(self): server_type="ASYNC_SERVER", client_processes=1, server_processes=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1209,7 +1216,7 @@ def scenarios(self): channels=1, client_processes=1, server_processes=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1239,7 +1246,7 @@ def scenarios(self): channels=1, client_processes=1, unconstrained_client="async", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1251,7 +1258,7 @@ def scenarios(self): channels=1, client_processes=0, server_language="c++", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1276,7 +1283,7 @@ def scenarios(self): channels=1, client_processes=1, server_processes=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -1296,7 +1303,7 @@ def scenarios(self): rpc_type="STREAMING", client_type="SYNC_CLIENT", server_type="SYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1304,7 +1311,7 @@ def scenarios(self): rpc_type="UNARY", client_type="SYNC_CLIENT", server_type="SYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1348,7 +1355,7 @@ def scenarios(self): server_type="SYNC_SERVER", req_size=1024 * 1024, resp_size=1024 * 1024, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -1471,7 +1478,10 @@ def scenarios(self): for secure in [True, False]: secstr = "secure" if secure else "insecure" - smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + smoketest_categories = ([SMOKETEST] if secure else []) + [ + SCALABLE, + SCALABLE_LIMITED, + ] yield _ping_pong_scenario( "java_generic_async_streaming_ping_pong_%s" % secstr, @@ -1524,7 +1534,7 @@ def scenarios(self): unconstrained_client="async", secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=smoketest_categories + [SCALABLE], + categories=smoketest_categories + [SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -1535,7 +1545,7 @@ def scenarios(self): unconstrained_client="async", secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -1547,7 +1557,7 @@ def scenarios(self): use_generic_payload=True, secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -1593,7 +1603,10 @@ def scenarios(self): for secure in [True, False]: secstr = "secure" if secure else "insecure" - smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + smoketest_categories = ([SMOKETEST] if secure else []) + [ + SCALABLE, + SCALABLE_LIMITED, + ] # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, # but that's mostly because of lack of better name of the enum value. @@ -1635,7 +1648,7 @@ def scenarios(self): server_type="SYNC_SERVER", unconstrained_client="async", secure=secure, - categories=smoketest_categories + [SCALABLE], + categories=smoketest_categories + [SCALABLE, SCALABLE_LIMITED], ) # unconstrained_client='async' is intended (client uses goroutines) @@ -1646,7 +1659,7 @@ def scenarios(self): server_type="SYNC_SERVER", unconstrained_client="async", secure=secure, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) # unconstrained_client='async' is intended (client uses goroutines) @@ -1660,7 +1673,7 @@ def scenarios(self): unconstrained_client="async", use_generic_payload=True, secure=secure, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) # TODO(jtattermusch): add scenarios go vs C++ @@ -1699,7 +1712,10 @@ def scenarios(self): for secure in [True, False]: secstr = "secure" if secure else "insecure" - smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + smoketest_categories = ([SMOKETEST] if secure else []) + [ + SCALABLE, + SCALABLE_LIMITED, + ] yield _ping_pong_scenario( "node_to_node_generic_async_streaming_ping_pong_%s" % secstr, @@ -1739,7 +1755,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", secure=secure, - categories=smoketest_categories + [SCALABLE], + categories=smoketest_categories + [SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -1750,7 +1766,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", secure=secure, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) yield _ping_pong_scenario( @@ -1762,7 +1778,7 @@ def scenarios(self): unconstrained_client="async", use_generic_payload=True, secure=secure, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_LIMITED], ) # TODO(murgatroid99): add scenarios node vs C++ diff --git a/tools/run_tests/performance/scenario_config_exporter.py b/tools/run_tests/performance/scenario_config_exporter.py index 436557f5691d37..7a783e200ae53c 100755 --- a/tools/run_tests/performance/scenario_config_exporter.py +++ b/tools/run_tests/performance/scenario_config_exporter.py @@ -192,6 +192,7 @@ def main() -> None: "all", "inproc", "scalable", + "scalable_limited", "smoketest", "sweep", "psm",