diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh index a8da8a9fae6a0..9cfe036a7b7ad 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh @@ -19,7 +19,6 @@ cd "$(dirname "$0")/../../.." source tools/internal_ci/helper_scripts/prepare_build_linux_rc - # Environment variables to select repos and branches for various repos. # You can edit these lines if you want to run from a fork. GRPC_CORE_REPO=grpc/grpc @@ -43,20 +42,20 @@ gcloud auth configure-docker # Connect to benchmarks-prod2 cluster. gcloud config set project grpc-testing gcloud container clusters get-credentials benchmarks-prod2 \ - --zone us-central1-b --project grpc-testing + --zone us-central1-b --project grpc-testing # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" # BEGIN differentiate experimental configuration from master configuration. if [[ "${KOKORO_BUILD_INITIATOR%%-*}" == kokoro && "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - # Use "official" BQ tables only for builds initiated by Kokoro and running - # from grpc/grpc. These results show up in the "official" public dashboard. - BIGQUERY_TABLE_8CORE=e2e_benchmarks.ci_master_results_8core - BIGQUERY_TABLE_32CORE=e2e_benchmarks.ci_master_results_32core + # Use "official" BQ tables only for builds initiated by Kokoro and running + # from grpc/grpc. These results show up in the "official" public dashboard. + BIGQUERY_TABLE_8CORE=e2e_benchmarks.ci_master_results_8core + BIGQUERY_TABLE_32CORE=e2e_benchmarks.ci_master_results_32core else - # Use experimental BQ tables otherwise. - BIGQUERY_TABLE_8CORE=e2e_benchmarks.experimental_results - BIGQUERY_TABLE_32CORE=e2e_benchmarks.experimental_results_32core + # Use experimental BQ tables otherwise. + BIGQUERY_TABLE_8CORE=e2e_benchmarks.experimental_results + BIGQUERY_TABLE_32CORE=e2e_benchmarks.experimental_results_32core fi # END differentiate experimental configuration from master configuration. CLOUD_LOGGING_URL="https://source.cloud.google.com/results/invocations/${KOKORO_BUILD_ID}" @@ -67,9 +66,9 @@ ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/" GRPC_COMMIT="$(git show --format="%H" --no-patch)" # Prebuilt workers for core languages are always built from grpc/grpc. if [[ "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" + GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" else - GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" + GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" fi GRPC_DOTNET_COMMIT="$(git ls-remote "https://github.com/${GRPC_DOTNET_REPO}.git" "${GRPC_DOTNET_GITREF}" | cut -f1)" @@ -94,30 +93,30 @@ popd # Build test configurations. buildConfigs() { - local -r pool="$1" - local -r table="$2" - shift 2 - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${table}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_COMMIT}" \ - -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ - -a ci_gitCommit_dotnet="${GRPC_DOTNET_COMMIT}" \ - -a ci_gitCommit_go="${GRPC_GO_COMMIT}" \ - -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=scalable \ - --allow_client_language=c++ --allow_server_language=c++ \ - --allow_server_language=node \ - -o "loadtest_with_prebuilt_workers_${pool}.yaml" + local -r pool="$1" + local -r table="$2" + shift 2 + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${table}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_COMMIT}" \ + -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ + -a ci_gitCommit_dotnet="${GRPC_DOTNET_COMMIT}" \ + -a ci_gitCommit_go="${GRPC_GO_COMMIT}" \ + -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ + -a pool="${pool}" --category=scalable \ + --allow_client_language=c++ --allow_server_language=c++ \ + --allow_server_language=node \ + -o "loadtest_with_prebuilt_workers_${pool}.yaml" } # Add languages @@ -126,9 +125,9 @@ declare -a configLangArgs32core=() declare -a runnerLangArgs=() # c++ -configLangArgs8core+=( -l c++ ) -configLangArgs32core+=( -l c++ ) -runnerLangArgs+=( -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +configLangArgs8core+=(-l c++) +configLangArgs32core+=(-l c++) +runnerLangArgs+=(-l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") # dotnet # configLangArgs8core+=( -l dotnet ) @@ -136,34 +135,34 @@ runnerLangArgs+=( -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) # runnerLangArgs+=( -l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}" ) # # go -configLangArgs8core+=( -l go ) -configLangArgs32core+=( -l go ) -runnerLangArgs+=( -l "go:${GRPC_GO_REPO}:${GRPC_GO_COMMIT}" ) +configLangArgs8core+=(-l go) +configLangArgs32core+=(-l go) +runnerLangArgs+=(-l "go:${GRPC_GO_REPO}:${GRPC_GO_COMMIT}") # java -configLangArgs8core+=( -l java ) -configLangArgs32core+=( -l java ) -runnerLangArgs+=( -l "java:${GRPC_JAVA_REPO}:${GRPC_JAVA_COMMIT}" ) +configLangArgs8core+=(-l java) +configLangArgs32core+=(-l java) +runnerLangArgs+=(-l "java:${GRPC_JAVA_REPO}:${GRPC_JAVA_COMMIT}") # node -configLangArgs8core+=( -l node ) # 8-core only. -runnerLangArgs+=( -l "node:${GRPC_NODE_REPO}:${GRPC_NODE_COMMIT}" ) +configLangArgs8core+=(-l node) # 8-core only. +runnerLangArgs+=(-l "node:${GRPC_NODE_REPO}:${GRPC_NODE_COMMIT}") # python -configLangArgs8core+=( -l python ) # 8-core only. -runnerLangArgs+=( -l "python:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +configLangArgs8core+=(-l python) # 8-core only. +runnerLangArgs+=(-l "python:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") # ruby -configLangArgs8core+=( -l ruby ) # 8-core only. -runnerLangArgs+=( -l "ruby:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +configLangArgs8core+=(-l ruby) # 8-core only. +runnerLangArgs+=(-l "ruby:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" "${configLangArgs8core[@]}" buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" "${configLangArgs32core[@]}" # Delete prebuilt images on exit. deleteImages() { - echo "deleting images on exit" - ../test-infra/bin/delete_prebuilt_workers \ + echo "deleting images on exit" + ../test-infra/bin/delete_prebuilt_workers \ -p "${PREBUILT_IMAGE_PREFIX}" \ -t "${UNIQUE_IDENTIFIER}" } @@ -171,16 +170,16 @@ trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers "${runnerLangArgs[@]}" \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" \ - -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" \ + -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" # Run tests. time ../test-infra/bin/runner \ - -i "loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -i "loadtest_with_prebuilt_workers_${WORKER_POOL_32CORE}.yaml" \ - -log-url-prefix "${LOG_URL_PREFIX}" \ - -polling-interval 5s \ - -delete-successful-tests \ - -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ - -o "runner/sponge_log.xml" + -i "loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -i "loadtest_with_prebuilt_workers_${WORKER_POOL_32CORE}.yaml" \ + -log-url-prefix "${LOG_URL_PREFIX}" \ + -polling-interval 5s \ + -delete-successful-tests \ + -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ + -o "runner/sponge_log.xml" diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke_cxx_experiments_framework.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke_cxx_experiments_framework.sh index dfad208da8c26..723369b25809b 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke_cxx_experiments_framework.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke_cxx_experiments_framework.sh @@ -18,9 +18,9 @@ set -ex # # To run the benchmarks, add your experiment to the set below. declare -a GRPC_EXPERIMENTS=( - "event_engine_listener" - "event_engine_client" - "event_engine_client,event_engine_listener" + "event_engine_listener" + "event_engine_client" + "event_engine_client,event_engine_listener" ) # Enter the gRPC repo root. @@ -43,13 +43,13 @@ gcloud auth configure-docker # Connect to benchmarks-prod2 cluster. gcloud config set project grpc-testing gcloud container clusters get-credentials benchmarks-prod2 \ - --zone us-central1-b --project grpc-testing + --zone us-central1-b --project grpc-testing # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" # BEGIN differentiate experimental configuration from master configuration. if [[ "${KOKORO_BUILD_INITIATOR%%-*}" == kokoro ]]; then - LOAD_TEST_PREFIX=kokoro + LOAD_TEST_PREFIX=kokoro fi # Use the "official" BQ tables so that the measurements will show up in the # "official" public dashboard. @@ -64,9 +64,9 @@ ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/" GRPC_COMMIT="$(git show --format="%H" --no-patch)" # Prebuilt workers for core languages are always built from grpc/grpc. if [[ "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" + GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" else - GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" + GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" fi # Kokoro jobs run on dedicated pools. @@ -89,62 +89,62 @@ declare -a loadtest_files=() # Build test configurations. buildConfigs() { - local -r pool="$1" - local -r base_table="$2" - local -r experiment="$3" - shift 3 - # Multiple experiments are delimited by `__` (two underscores) in BigQuery. - SANITIZED_EXPERIMENT="${experiment//,/__}" - OUTFILE="loadtest_with_prebuilt_workers_${pool}_${SANITIZED_EXPERIMENT}.yaml" - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_cxx_experiments.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${base_table}_${SANITIZED_EXPERIMENT}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -s grpc_experiment="${experiment}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_COMMIT}" \ - -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=dashboard \ - --allow_client_language=c++ --allow_server_language=c++ \ - --allow_server_language=node \ - -o "${OUTFILE}" - - loadtest_files+=(-i "${OUTFILE}") + local -r pool="$1" + local -r base_table="$2" + local -r experiment="$3" + shift 3 + # Multiple experiments are delimited by `__` (two underscores) in BigQuery. + SANITIZED_EXPERIMENT="${experiment//,/__}" + OUTFILE="loadtest_with_prebuilt_workers_${pool}_${SANITIZED_EXPERIMENT}.yaml" + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_cxx_experiments.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${base_table}_${SANITIZED_EXPERIMENT}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -s grpc_experiment="${experiment}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_COMMIT}" \ + -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ + -a pool="${pool}" --category=dashboard \ + --allow_client_language=c++ --allow_server_language=c++ \ + --allow_server_language=node \ + -o "${OUTFILE}" + + loadtest_files+=(-i "${OUTFILE}") } for experiment in "${GRPC_EXPERIMENTS[@]}"; do - buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" "${experiment}" -l c++ - buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" "${experiment}" -l c++ + buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" "${experiment}" -l c++ + buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" "${experiment}" -l c++ done # Delete prebuilt images on exit. deleteImages() { - echo "deleting images on exit" - ../test-infra/bin/delete_prebuilt_workers \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" + echo "deleting images on exit" + ../test-infra/bin/delete_prebuilt_workers \ + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" } trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers \ - -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" \ - -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" + -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" \ + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" \ + -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" # Run tests. ../test-infra/bin/runner \ - ${loadtest_files[@]} \ - -log-url-prefix "${LOG_URL_PREFIX}" \ - -polling-interval 5s \ - -delete-successful-tests \ - -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ - -o "runner/sponge_log.xml" + "${loadtest_files[@]}" \ + -log-url-prefix "${LOG_URL_PREFIX}" \ + -polling-interval 5s \ + -delete-successful-tests \ + -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ + -o "runner/sponge_log.xml" diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh index c3b99af80ebb6..e46c39addd97f 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh @@ -19,7 +19,6 @@ cd "$(dirname "$0")/../../.." source tools/internal_ci/helper_scripts/prepare_build_linux_rc - # Environment variables to select repos and branches for various repos. # You can edit these lines if you want to run from a fork. GRPC_CORE_REPO=grpc/grpc @@ -43,13 +42,13 @@ gcloud auth configure-docker # Connect to benchmarks-prod2 cluster. gcloud config set project grpc-testing gcloud container clusters get-credentials benchmarks-prod2 \ - --zone us-central1-b --project grpc-testing + --zone us-central1-b --project grpc-testing # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" # BEGIN differentiate experimental configuration from master configuration. if [[ "${KOKORO_BUILD_INITIATOR%%-*}" == kokoro ]]; then - LOAD_TEST_PREFIX=kokoro-test + LOAD_TEST_PREFIX=kokoro-test fi BIGQUERY_TABLE_8CORE=e2e_benchmarks.experimental_results BIGQUERY_TABLE_32CORE=e2e_benchmarks.experimental_results_32core @@ -62,9 +61,9 @@ ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/" GRPC_COMMIT="$(git show --format="%H" --no-patch)" # Prebuilt workers for core languages are always built from grpc/grpc. if [[ "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" + GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" else - GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" + GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" fi GRPC_DOTNET_COMMIT="$(git ls-remote "https://github.com/${GRPC_DOTNET_REPO}.git" "${GRPC_DOTNET_GITREF}" | cut -f1)" @@ -89,29 +88,29 @@ popd # Build test configurations. buildConfigs() { - local -r pool="$1" - local -r table="$2" - shift 2 - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${table}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_COMMIT}" \ - -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ - -a ci_gitCommit_dotnet="${GRPC_DOTNET_COMMIT}" \ - -a ci_gitCommit_go="${GRPC_GO_COMMIT}" \ - -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=scalable \ - --allow_client_language=c++ --allow_server_language=c++ \ - -o "loadtest_with_prebuilt_workers_${pool}.yaml" + local -r pool="$1" + local -r table="$2" + shift 2 + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${table}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_COMMIT}" \ + -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ + -a ci_gitCommit_dotnet="${GRPC_DOTNET_COMMIT}" \ + -a ci_gitCommit_go="${GRPC_GO_COMMIT}" \ + -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ + -a pool="${pool}" --category=scalable \ + --allow_client_language=c++ --allow_server_language=c++ \ + -o "loadtest_with_prebuilt_workers_${pool}.yaml" } # Add languages @@ -120,9 +119,9 @@ declare -a configLangArgs32core=() declare -a runnerLangArgs=() # c++ -configLangArgs8core+=( -l c++ ) -configLangArgs32core+=( -l c++ ) -runnerLangArgs+=( -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +configLangArgs8core+=(-l c++) +configLangArgs32core+=(-l c++) +runnerLangArgs+=(-l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") # dotnet # configLangArgs8core+=( -l dotnet ) @@ -130,34 +129,34 @@ runnerLangArgs+=( -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) # runnerLangArgs+=( -l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}" ) # # go -configLangArgs8core+=( -l go ) -configLangArgs32core+=( -l go ) -runnerLangArgs+=( -l "go:${GRPC_GO_REPO}:${GRPC_GO_COMMIT}" ) +configLangArgs8core+=(-l go) +configLangArgs32core+=(-l go) +runnerLangArgs+=(-l "go:${GRPC_GO_REPO}:${GRPC_GO_COMMIT}") # java -configLangArgs8core+=( -l java ) -configLangArgs32core+=( -l java ) -runnerLangArgs+=( -l "java:${GRPC_JAVA_REPO}:${GRPC_JAVA_COMMIT}" ) +configLangArgs8core+=(-l java) +configLangArgs32core+=(-l java) +runnerLangArgs+=(-l "java:${GRPC_JAVA_REPO}:${GRPC_JAVA_COMMIT}") # node -configLangArgs8core+=( -l node ) # 8-core only. -runnerLangArgs+=( -l "node:${GRPC_NODE_REPO}:${GRPC_NODE_COMMIT}" ) +configLangArgs8core+=(-l node) # 8-core only. +runnerLangArgs+=(-l "node:${GRPC_NODE_REPO}:${GRPC_NODE_COMMIT}") # python -configLangArgs8core+=( -l python ) # 8-core only. -runnerLangArgs+=( -l "python:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +configLangArgs8core+=(-l python) # 8-core only. +runnerLangArgs+=(-l "python:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") # ruby -configLangArgs8core+=( -l ruby ) # 8-core only. -runnerLangArgs+=( -l "ruby:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +configLangArgs8core+=(-l ruby) # 8-core only. +runnerLangArgs+=(-l "ruby:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" "${configLangArgs8core[@]}" buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" "${configLangArgs32core[@]}" # Delete prebuilt images on exit. deleteImages() { - echo "deleting images on exit" - ../test-infra/bin/delete_prebuilt_workers \ + echo "deleting images on exit" + ../test-infra/bin/delete_prebuilt_workers \ -p "${PREBUILT_IMAGE_PREFIX}" \ -t "${UNIQUE_IDENTIFIER}" } @@ -165,16 +164,16 @@ trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers "${runnerLangArgs[@]}" \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" \ - -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" \ + -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" # Run tests. time ../test-infra/bin/runner \ - -i "loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -i "loadtest_with_prebuilt_workers_${WORKER_POOL_32CORE}.yaml" \ - -log-url-prefix "${LOG_URL_PREFIX}" \ - -polling-interval 5s \ - -delete-successful-tests \ - -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ - -o "runner/sponge_log.xml" + -i "loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -i "loadtest_with_prebuilt_workers_${WORKER_POOL_32CORE}.yaml" \ + -log-url-prefix "${LOG_URL_PREFIX}" \ + -polling-interval 5s \ + -delete-successful-tests \ + -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ + -o "runner/sponge_log.xml" diff --git a/tools/internal_ci/linux/grpc_psm_performance_gke_experiment.sh b/tools/internal_ci/linux/grpc_psm_performance_gke_experiment.sh index 143e923359bd9..44a67643838ed 100644 --- a/tools/internal_ci/linux/grpc_psm_performance_gke_experiment.sh +++ b/tools/internal_ci/linux/grpc_psm_performance_gke_experiment.sh @@ -27,13 +27,13 @@ gcloud auth configure-docker # Connect to benchmarks-prod2 cluster. gcloud config set project grpc-testing gcloud container clusters get-credentials psm-benchmarks-performance \ - --zone us-central1-b --project grpc-testing + --zone us-central1-b --project grpc-testing # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" # BEGIN differentiate experimental configuration from master configuration. if [[ "${KOKORO_BUILD_INITIATOR%%-*}" == kokoro ]]; then - LOAD_TEST_PREFIX=kokoro-test + LOAD_TEST_PREFIX=kokoro-test fi BIGQUERY_TABLE_8CORE=e2e_benchmarks.psm_experimental_results_8core # END differentiate experimental configuration from master configuration. @@ -45,9 +45,9 @@ ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/" GRPC_GITREF="$(git show --format="%H" --no-patch)" # Prebuilt workers for core languages are always built from grpc/grpc. if [[ "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - GRPC_CORE_GITREF="${KOKORO_GIT_COMMIT}" + GRPC_CORE_GITREF="${KOKORO_GIT_COMMIT}" else - GRPC_CORE_GITREF="$(git ls-remote -h https://github.com/grpc/grpc.git master | cut -f1)" + GRPC_CORE_GITREF="$(git ls-remote -h https://github.com/grpc/grpc.git master | cut -f1)" fi GRPC_JAVA_GITREF="$(git ls-remote -h https://github.com/grpc/grpc-java.git master | cut -f1)" # Kokoro jobs run on dedicated pools. @@ -75,69 +75,69 @@ PSM_IMAGE_TAG=${TEST_INFRA_VERSION} # Build psm test configurations. psmBuildConfigs() { - local -r pool="$1" - local -r table="$2" - local -r proxy_type="$3" - - shift 3 - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_psm_"${proxy_type}"_prebuilt_all_languages.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${table}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -s psm_image_prefix="${PSM_IMAGE_PREFIX}" \ - -s psm_image_tag="${PSM_IMAGE_TAG}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_GITREF}" \ - -a ci_gitCommit_java="${GRPC_JAVA_GITREF}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - -a enablePrometheus=true \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" -u "${proxy_type}"\ - -a pool="${pool}" \ - --category=psm \ - --allow_client_language=c++ --allow_server_language=c++ \ - -o "psm_${proxy_type}_loadtest_with_prebuilt_workers_${pool}.yaml" + local -r pool="$1" + local -r table="$2" + local -r proxy_type="$3" + + shift 3 + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_psm_"${proxy_type}"_prebuilt_all_languages.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${table}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -s psm_image_prefix="${PSM_IMAGE_PREFIX}" \ + -s psm_image_tag="${PSM_IMAGE_TAG}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_GITREF}" \ + -a ci_gitCommit_java="${GRPC_JAVA_GITREF}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + -a enablePrometheus=true \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" -u "${proxy_type}" \ + -a pool="${pool}" \ + --category=psm \ + --allow_client_language=c++ --allow_server_language=c++ \ + -o "psm_${proxy_type}_loadtest_with_prebuilt_workers_${pool}.yaml" } -psmBuildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" proxied -a queue="${WORKER_POOL_8CORE}-proxied" -l c++ -l java --client_channels=8 --server_threads=16 --offered_loads 100 300 500 700 900 1000 1500 2000 2500 4000 6000 8000 10000 12000 14000 16000 18000 20000 22000 24000 26000 28000 30000 +psmBuildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" proxied -a queue="${WORKER_POOL_8CORE}-proxied" -l c++ -l java --client_channels=8 --server_threads=16 --offered_loads 100 300 500 700 900 1000 1500 2000 2500 4000 6000 8000 10000 12000 14000 16000 18000 20000 22000 24000 26000 28000 30000 psmBuildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" proxyless -a queue="${WORKER_POOL_8CORE}-proxyless" -l c++ -l java --client_channels=8 --server_threads=16 --offered_loads 100 300 500 700 900 1000 1500 2000 2500 4000 6000 8000 10000 12000 14000 16000 18000 20000 22000 24000 26000 28000 30000 # Build regular test configurations. buildConfigs() { - local -r pool="$1" - local -r table="$2" - shift 2 - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${table}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_GITREF}" \ - -a ci_gitCommit_java="${GRPC_JAVA_GITREF}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - -a enablePrometheus=true \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=psm \ - --allow_client_language=c++ --allow_server_language=c++ \ - -o "regular_loadtest_with_prebuilt_workers_${pool}.yaml" + local -r pool="$1" + local -r table="$2" + shift 2 + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${table}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_GITREF}" \ + -a ci_gitCommit_java="${GRPC_JAVA_GITREF}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + -a enablePrometheus=true \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ + -a pool="${pool}" --category=psm \ + --allow_client_language=c++ --allow_server_language=c++ \ + -o "regular_loadtest_with_prebuilt_workers_${pool}.yaml" } buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -a queue="${WORKER_POOL_8CORE}-regular" -l c++ -l java --client_channels=8 --server_threads=16 --offered_loads 100 300 500 700 900 1000 1500 2000 2500 4000 6000 8000 10000 12000 14000 16000 18000 20000 22000 24000 26000 28000 30000 # Delete prebuilt images on exit. deleteImages() { - echo "deleting images on exit" - ../test-infra/bin/delete_prebuilt_workers \ + echo "deleting images on exit" + ../test-infra/bin/delete_prebuilt_workers \ -p "${PREBUILT_IMAGE_PREFIX}" \ -t "${UNIQUE_IDENTIFIER}" } @@ -145,22 +145,22 @@ trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers \ - -l "cxx:${GRPC_CORE_GITREF}" \ - -l "java:${GRPC_JAVA_GITREF}" \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" \ - -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" + -l "cxx:${GRPC_CORE_GITREF}" \ + -l "java:${GRPC_JAVA_GITREF}" \ + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" \ + -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" # Run tests. time ../test-infra/bin/runner \ - -i "psm_proxied_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -i "psm_proxyless_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -i "regular_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -log-url-prefix "${LOG_URL_PREFIX}" \ - -annotation-key queue \ - -polling-interval 5s \ - -delete-successful-tests \ - -c "${WORKER_POOL_8CORE}-proxied:1" \ - -c "${WORKER_POOL_8CORE}-proxyless:1" \ - -c "${WORKER_POOL_8CORE}-regular:1" \ - -o "runner/sponge_log.xml" + -i "psm_proxied_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -i "psm_proxyless_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -i "regular_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -log-url-prefix "${LOG_URL_PREFIX}" \ + -annotation-key queue \ + -polling-interval 5s \ + -delete-successful-tests \ + -c "${WORKER_POOL_8CORE}-proxied:1" \ + -c "${WORKER_POOL_8CORE}-proxyless:1" \ + -c "${WORKER_POOL_8CORE}-regular:1" \ + -o "runner/sponge_log.xml"