[DEVOPS-3419] test automation: Update YBC for DB unit testing when version changes

Summary:
Split out YBC stable version to separate conf file, so that a version update will trigger DB unit
testing, but other changes to reference.conf need not trigger such DB CI testing.

Fix prep_ybc_testing() to notice when the version installed in build/ybc is not correct and
re-install.

Test Plan:
Running single test to verify correct updates to build/ybc when needed, and whether or not build/ybc
exists, and whether or not build/ybc/ybc_version.conf exists.
Also that new downloads to /opt/yb-build/ybc staging directory works.
./yb_build.sh --sb --java-test 'org.yb.pgsql.TestYbBackup#testMixedLegacyColocatedDatabase'

Reviewers: dshubin, mlillibridge, jharveysmith

Reviewed By: dshubin, jharveysmith

Subscribers: devops

Differential Revision: https://phorge.dev.yugabyte.com/D44369

Author: svarnau

build-support/common-test-env.sh

@@ -2102,20 +2102,21 @@ prep_ybc_testing() {
   # YBC is supported only on linux.
   if is_linux; then
     ybc_dest="$YB_SRC_ROOT/build/ybc"
-    if [[ -d "${ybc_dest}" ]]; then
-      log "Found existing $ybc_dest directory, skipping YBC prep."
+    config_file="$YB_SRC_ROOT/managed/src/main/resources/ybc_version.conf"
+    if [[ -d "${ybc_dest}" ]] && diff "$config_file" "${ybc_dest}/ybc_version.conf"; then
+        log "Found up to date $ybc_dest directory, skipping YBC prep."
     else
       ybc_tarball_dir="/opt/yb-build/ybc"
-      config_file="$YB_SRC_ROOT/managed/src/main/resources/reference.conf"
       # check that current version is downloaded
       "$YB_SRC_ROOT"/managed/download_ybc.sh -i -c "$config_file" -d "$ybc_tarball_dir"
       # Extract version the same way as download_ybc.sh does.
-      ybc_version=$(grep ybc -A2 "${config_file}" |
-                    awk -F '= ' '/stable_version/ {print $2}' | tr -d \")
+      ybc_version=$(awk -F '= ' '/stable_version/ {print $2}' "${config_file}" | tr -d \")
       ybc_tar=$(compgen -G "${ybc_tarball_dir}/ybc-${ybc_version}-*-${YB_TARGET_ARCH}.tar.gz")
       log "Unpacking ${ybc_tar} bin/ binaries to ${ybc_dest}/"
+      rm -rf "${ybc_dest}"
       mkdir -p "${ybc_dest}"
       tar -x -f "${ybc_tar}" -C "${ybc_dest}" --strip-components=2 --wildcards 'yb*/bin/*'
+      cp "$config_file" "${ybc_dest}/ybc_version.conf"
     fi
   fi
 }

managed/build.sbt

@@ -389,7 +389,7 @@ versionGenerate := {
     (Compile / resourceDirectory).value / "version_metadata.json").!
   ybLog("version_metadata.json Generated")
   Process("rm -f " + (Compile / resourceDirectory).value / "gen_version_info.log").!
-  var downloadYbcCmd = "./download_ybc.sh -c " + (Compile / resourceDirectory).value / "reference.conf" + " -i"
+  var downloadYbcCmd = "./download_ybc.sh -c " + (Compile / resourceDirectory).value / "ybc_version.conf" + " -i"
   if (moveYbcPackage) {
     downloadYbcCmd = downloadYbcCmd + " -s"
   }

managed/download_ybc.sh

@@ -57,7 +57,7 @@ while [[ $# -gt 0 ]]; do
 done
 
 ybc_version=$(
-  grep ybc -A2 ${config_file} |  awk -F '= ' '/stable_version/ {print $2}' | tr -d \"
+  awk -F '= ' '/stable_version/ {print $2}' ${config_file} | tr -d \"
 )
 
 # Use http, so aws s3 command/credential setup is not required.

managed/src/main/resources/reference.conf

@@ -1447,7 +1447,7 @@ yb {
 
 ybc {
   releases {
-    stable_version = "2.2.0.2-b5"
+    include "ybc_version.conf"
     path = "/opt/yugabyte/ybc/releases"
   }
   compatible_db_version = "2.15.0.0-b1"

managed/src/main/resources/ybc_version.conf

@@ -0,0 +1,3 @@
+{
+  stable_version = "2.2.0.2-b5"
+}