Merge branch 'master' into PYTHON-4521

Author: NoahStapp

.evergreen/config.yml

@@ -365,7 +365,7 @@ functions:
           ${PREPARE_SHELL}
           set -o xtrace
           export PYTHON_BINARY=${PYTHON_BINARY}
-          bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-mockupdb
+          bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-mockupdb
 
   "run doctests":
      - command: shell.exec
@@ -375,18 +375,19 @@ functions:
          script: |
            ${PREPARE_SHELL}
            set -o xtrace
-           PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m doc-test
+           PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test
 
   "run tests":
     - command: shell.exec
       params:
         working_dir: "src"
         shell: bash
-        include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+        background: true
+        include_expansions_in_env:  ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
         script: |
           ${PREPARE_SHELL}
           if [ -n "${test_encryption}" ]; then
-            . .evergreen/tox.sh -m setup-encryption
+            ./.evergreen/hatch.sh encryption:setup
           fi
     - command: shell.exec
       type: test
@@ -407,6 +408,7 @@ functions:
           if [ -n "${test_encryption}" ]; then
             # Disable xtrace (just in case it was accidentally set).
             set +x
+            bash ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh
             export TEST_ENCRYPTION=1
             if [ -n "${test_encryption_pyopenssl}" ]; then
               export TEST_ENCRYPTION_PYOPENSSL=1
@@ -446,7 +448,7 @@ functions:
             SSL=${SSL} \
             TEST_DATA_LAKE=${TEST_DATA_LAKE} \
             MONGODB_API_VERSION=${MONGODB_API_VERSION} \
-            bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg
+            bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
 
   "run enterprise auth tests":
     - command: shell.exec
@@ -462,7 +464,7 @@ functions:
             PYTHON_BINARY="${PYTHON_BINARY}" \
             TEST_ENTERPRISE_AUTH=1 \
             AUTH=auth \
-            bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg
+            bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
 
   "run atlas tests":
     - command: shell.exec
@@ -478,7 +480,7 @@ functions:
           PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \
             PYTHON_BINARY="${PYTHON_BINARY}" \
             TEST_ATLAS=1 \
-            bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg
+            bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
 
   "get aws auth secrets":
     - command: subprocess.exec
@@ -597,7 +599,7 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          . .evergreen/tox.sh -m teardown-encryption
+          . .evergreen/hatch.sh encryption:teardown
           rm -rf $DRIVERS_TOOLS || true
           rm -f ./secrets-export.sh || true
 
@@ -683,7 +685,7 @@ functions:
           PYTHON_BINARY=${PYTHON_BINARY} \
           CA_FILE="$DRIVERS_TOOLS/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
           OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
-          bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg
+          bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
 
   run-valid-ocsp-server:
     - command: shell.exec
@@ -2128,7 +2130,7 @@ tasks:
               ${PREPARE_SHELL}
               export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
               export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/${bucket_name}/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz
-              SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/tox.sh -m test-eg
+              SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg
 
     - name: testazurekms-task
       commands:

.evergreen/hatch.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -o errexit  # Exit the script with error if any of the commands fail
+set -x
+
+. .evergreen/utils.sh
+
+if [ -z "$PYTHON_BINARY" ]; then
+    PYTHON_BINARY=$(find_python3)
+fi
+
+if $PYTHON_BINARY -m hatch --version; then
+    run_hatch() {
+      $PYTHON_BINARY hatch run "$@"
+    }
+else # No toolchain hatch present, set up virtualenv before installing hatch
+    createvirtualenv "$PYTHON_BINARY" hatchenv
+    trap "deactivate; rm -rf hatchenv" EXIT HUP
+    python -m pip install -q hatch
+    run_hatch() {
+      python -m hatch run "$@"
+    }
+fi
+
+run_hatch "${@:1}"

.evergreen/run-azurekms-fail-test.sh

@@ -7,4 +7,4 @@ PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \
     KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
     LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz \
     SUCCESS=false TEST_FLE_AZURE_AUTO=1 \
-    ./.evergreen/tox.sh -m test-eg
+    ./.evergreen/hatch.sh test:test-eg

.evergreen/run-azurekms-test.sh

@@ -16,6 +16,6 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/tox.sh -m test-eg" \
+AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/hatch.sh test:test-eg" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 echo "Running test ... end"

.evergreen/run-gcpkms-test.sh

@@ -14,5 +14,5 @@ echo "Untarring file ... begin"
 GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/tox.sh -m test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/hatch.sh test:test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Running test ... end"

.evergreen/run-mongodb-aws-ecs-test.sh

@@ -30,5 +30,5 @@ export TEST_AUTH_AWS=1
 export AUTH="auth"
 export SET_XTRACE_ON=1
 cd src
-$PYTHON_BINARY -m pip install -q --user tox
-bash .evergreen/tox.sh -m test-eg
+$PYTHON_BINARY -m pip install -q --user hatch
+bash .evergreen/hatch.sh test:test-eg

.evergreen/run-mongodb-aws-test.sh

@@ -24,4 +24,4 @@ set -x
 export TEST_AUTH_AWS=1
 export AUTH="auth"
 export SET_XTRACE_ON=1
-bash ./.evergreen/tox.sh -m test-eg
+bash ./.evergreen/hatch.sh test:test-eg

.evergreen/run-mongodb-oidc-test.sh

@@ -29,4 +29,4 @@ fi
 export TEST_AUTH_OIDC=1
 export COVERAGE=1
 export AUTH="auth"
-bash ./.evergreen/tox.sh -m test-eg -- "${@:1}"
+bash ./.evergreen/hatch.sh test:test-eg -- "${@:1}"

.evergreen/run-perf-tests.sh

@@ -16,4 +16,4 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json"
 export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
 export PERF_TEST=1
 
-bash ./.evergreen/tox.sh -m test-eg
+bash ./.evergreen/hatch.sh test:test-eg

.evergreen/run-tests.sh

@@ -30,6 +30,7 @@ set -o xtrace
 
 AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
+TEST_SUITES="test/ test/asynchronous/"
 TEST_ARGS="${*:1}"
 
 export PIP_QUIET=1  # Quiet by default
@@ -95,7 +96,7 @@ if [ -n "$TEST_LOADBALANCER" ]; then
     export LOAD_BALANCER=1
     export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI:-mongodb://127.0.0.1:8000/?loadBalanced=true}"
     export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI:-mongodb://127.0.0.1:8001/?loadBalanced=true}"
-    export TEST_ARGS="test/test_load_balancer.py"
+    export TEST_SUITES="test/test_load_balancer.py"
 fi
 
 if [ "$SSL" != "nossl" ]; then
@@ -123,9 +124,9 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE
 
     python -m pip install '.[encryption]'
 
-    # Install libmongocrypt if necessary.
+    # Setup encryption if necessary.
     if [ ! -d "libmongocrypt" ]; then
-        bash ./.evergreen/setup-libmongocrypt.sh
+        bash ./.evergreen/setup-encryption.sh
     fi
 
     # Use the nocrypto build to avoid dependency issues with older windows/python versions.
@@ -171,9 +172,7 @@ if [ -n "$TEST_ENCRYPTION" ]; then
         export PATH=$CRYPT_SHARED_DIR:$PATH
     fi
     # Only run the encryption tests.
-    if [ -z "$TEST_ARGS" ]; then
-        TEST_ARGS="test/test_encryption.py"
-    fi
+    TEST_SUITES="test/test_encryption.py"
 fi
 
 if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
@@ -187,9 +186,7 @@ if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
       exit 1
     fi
 
-    if [ -z "$TEST_ARGS" ]; then
-        TEST_ARGS="test/test_on_demand_csfle.py"
-    fi
+    TEST_SUITES="test/test_on_demand_csfle.py"
 fi
 
 if [ -n "$TEST_INDEX_MANAGEMENT" ]; then
@@ -198,36 +195,36 @@ if [ -n "$TEST_INDEX_MANAGEMENT" ]; then
     set +x
     export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}"
     set -x
-    TEST_ARGS="test/test_index_management.py"
+    TEST_SUITES="test/test_index_management.py"
 fi
 
 if [ -n "$TEST_DATA_LAKE" ] && [ -z "$TEST_ARGS" ]; then
-    TEST_ARGS="test/test_data_lake.py"
+    TEST_SUITES="test/test_data_lake.py"
 fi
 
 if [ -n "$TEST_ATLAS" ]; then
-    TEST_ARGS="test/atlas/test_connection.py"
+    TEST_SUITES="test/atlas/test_connection.py"
 fi
 
 if [ -n "$TEST_OCSP" ]; then
     python -m pip install ".[ocsp]"
-    TEST_ARGS="test/ocsp/test_ocsp.py"
+    TEST_SUITES="test/ocsp/test_ocsp.py"
 fi
 
 if [ -n "$TEST_AUTH_AWS" ]; then
     python -m pip install ".[aws]"
-    TEST_ARGS="test/auth_aws/test_auth_aws.py"
+    TEST_SUITES="test/auth_aws/test_auth_aws.py"
 fi
 
 if [ -n "$TEST_AUTH_OIDC" ]; then
     python -m pip install ".[aws]"
-    TEST_ARGS="test/auth_oidc/test_auth_oidc.py $TEST_ARGS"
+    TEST_SUITES="test/auth_oidc/test_auth_oidc.py $TEST_ARGS"
 fi
 
 if [ -n "$PERF_TEST" ]; then
     python -m pip install simplejson
     start_time=$(date +%s)
-    TEST_ARGS="test/performance/perf_test.py"
+    TEST_SUITES="test/performance/perf_test.py"
 fi
 
 echo "Running $AUTH tests over $SSL with python $(which python)"
@@ -257,8 +254,7 @@ PIP_QUIET=0 python -m pip list
 if [ -z "$GREEN_FRAMEWORK" ]; then
     # Use --capture=tee-sys so pytest prints test output inline:
     # https://docs.pytest.org/en/stable/how-to/capture-stdout-stderr.html
-    python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 $TEST_ARGS
-    python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 test/asynchronous/ $TEST_ARGS
+    python -m pytest -v --capture=tee-sys --durations=5 --maxfail=10 $TEST_SUITES $TEST_ARGS
 else
     python green_framework_test.py $GREEN_FRAMEWORK -v $TEST_ARGS
 fi