Chore: Address Clickhouse Cloud test flakiness (#3782)

Author: erindru

.circleci/manage-test-db.sh

@@ -97,12 +97,16 @@ bigquery_init() {
     echo "$BIGQUERY_KEYFILE_CONTENTS" > $BIGQUERY_KEYFILE
 }
 
-bigquery_up() {
-    echo "BigQuery doesnt support creating databases"
-}
 
-bigquery_down() {
-    echo "BigQuery doesnt support dropping databases"
+# Clickhouse cloud
+clickhouse-cloud_init() {
+    # note: the ping endpoint doesnt seem to need any API keys
+    until curl https://$CLICKHOUSE_CLOUD_HOST:8443/ping
+    do
+        echo "Pinging Clickhouse Cloud service to ensure it's not in idle mode..."
+        sleep 5
+    done
+    echo "Clickhouse Cloud instance $CLICKHOUSE_CLOUD_HOST is up and running"
 }
 
 INIT_FUNC="${ENGINE}_init"
@@ -118,10 +122,10 @@ fi
 echo "Initializing $ENGINE"
 $INIT_FUNC
 
-if [ "$DIRECTION" == "up" ]; then
+if [ "$DIRECTION" == "up" ] && function_exists $UP_FUNC; then
     echo "Creating database $DB_NAME"
     $UP_FUNC $DB_NAME
-elif [ "$DIRECTION" == "down" ]; then
+elif [ "$DIRECTION" == "down" ] && function_exists $DOWN_FUNC; then
     echo "Dropping database $DB_NAME"
     $DOWN_FUNC $DB_NAME
 fi

Makefile

@@ -211,7 +211,7 @@ redshift-test: guard-REDSHIFT_HOST guard-REDSHIFT_USER guard-REDSHIFT_PASSWORD g
 	pytest -n auto -x -m "redshift" --retries 3 --junitxml=test-results/junit-redshift.xml
 
 clickhouse-cloud-test: guard-CLICKHOUSE_CLOUD_HOST guard-CLICKHOUSE_CLOUD_USERNAME guard-CLICKHOUSE_CLOUD_PASSWORD engine-clickhouse-install
-	pytest -n auto -x -m "clickhouse_cloud" --retries 3 --junitxml=test-results/junit-clickhouse-cloud.xml
+	pytest -n 1 -m "clickhouse_cloud" --retries 3 --junitxml=test-results/junit-clickhouse-cloud.xml
 
 athena-test: guard-AWS_ACCESS_KEY_ID guard-AWS_SECRET_ACCESS_KEY guard-ATHENA_S3_WAREHOUSE_LOCATION engine-athena-install
 	pytest -n auto -x -m "athena" --retries 3 --retry-delay 10 --junitxml=test-results/junit-athena.xml

tests/core/engine_adapter/integration/config.yaml

@@ -149,6 +149,9 @@ gateways:
       port: 8443
       username: {{ env_var("CLICKHOUSE_CLOUD_USERNAME") }}
       password: {{ env_var("CLICKHOUSE_CLOUD_PASSWORD") }}
+      connect_timeout: 30
+      connection_pool_options:
+        retries: 5
     state_connection:
       type: duckdb