Project import generated by Copybara. (#88)

GitOrigin-RevId: 751f99dbec47b18cd9365114950cd2346501014a

Co-authored-by: Snowflake Authors <[email protected]>

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Release History
 
+## 1.2.2
+
+### Bug Fixes
+
+### Behavior Changes
+
+### New Features
+
+- Model Registry: Support providing external access integrations when deploying a model to SPCS. This will help and be
+  required to make sure the deploying process work as long as SPCS will by default deny all network connections. The
+  following endpoints must be allowed to make deployment work: docker.com:80, docker.com:443, anaconda.com:80,
+  anaconda.com:443, anaconda.org:80, anaconda.org:443, pypi.org:80, pypi.org:443. If you are using
+  `snowflake.ml.model.models.huggingface_pipeline.HuggingFacePipelineModel` object, the following endpoints are required
+  to be allowed: huggingface.com:80, huggingface.com:443, huggingface.co:80, huggingface.co:443.
+
 ## 1.2.1
 
 ### New Features

bazel/py_rules.bzl

@@ -259,6 +259,7 @@ def _py_wheel_impl(ctx):
             "--outdir",
             wheel_output_dir.path,
         ],
+        use_default_shell_env = True,
         progress_message = "Building Wheel",
         mnemonic = "WheelBuild",
     )

ci/RunBazelAction.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # DESCRIPTION: Utility Shell script to run bazel action for snowml repository
 #
-# RunBazelAction.sh <test|coverage> [-b <bazel_path>] [-m merge_gate|continuous_run|local_unittest|local_all] [-t <target>] [-c <path_to_coverage_report>]
+# RunBazelAction.sh <test|coverage> [-b <bazel_path>] [-m merge_gate|continuous_run|quarantined|local_unittest|local_all] [-t <target>] [-c <path_to_coverage_report>]
 #
 # Args:
 #   action: bazel action, choose from test and coverage
@@ -10,11 +10,13 @@
 #   -b: specify path to bazel.
 #   -m: specify the mode from the following options
 #       merge_gate: run affected tests only.
-#       continuous_run (default): run all tests except auto-generated tests. (For nightly run.)
+#       continuous_run (default): run all tests. (For nightly run. Alias: release)
+#       quarantined: Run quarantined tests.
 #       local_unit: run all unit tests affected by target defined by -t
 #       local_all: run all tests including integration tests affected by target defined by -t
 #   -t: specify the target for local_unit and local_all mode
 #   -c: specify the path to the coverage report dat file.
+#   -e: specify the environment, used to determine.
 #
 
 set -o pipefail
@@ -24,25 +26,29 @@ set -e
 bazel="bazel"
 mode="continuous_run"
 target=""
+SF_ENV="prod3"
 PROG=$0
 
 action=$1 && shift
 
 help() {
     local exit_code=$1
-    echo "Usage: ${PROG} <test|coverage> [-b <bazel_path>] [-m merge_gate|continuous_run|local_unittest|local_all]"
+    echo "Usage: ${PROG} <test|coverage> [-b <bazel_path>] [-m merge_gate|continuous_run|quarantined|local_unittest|local_all] [-e <snowflake_env>]"
     exit "${exit_code}"
 }
 
 if [[ "${action}" != "test" && "${action}" != "coverage" ]]; then
     help 1
 fi
 
-while getopts "b:m:t:c:h" opt; do
+while getopts "b:m:t:c:e:h" opt; do
     case "${opt}" in
     m)
-        if [[ "${OPTARG}" = "merge_gate" || "${OPTARG}" = "continuous_run" || "${OPTARG}" = "local_unittest" || "${OPTARG}" = "local_all" ]]; then
+        if [[ "${OPTARG}" = "merge_gate" || "${OPTARG}" = "continuous_run" || "${OPTARG}" = "quarantined" || "${OPTARG}" = "release" || "${OPTARG}" = "local_unittest" || "${OPTARG}" = "local_all" ]]; then
             mode="${OPTARG}"
+            if [[ $mode = "release" ]]; then
+                mode="continuous_run"
+            fi
         else
             help 1
         fi
@@ -60,6 +66,9 @@ while getopts "b:m:t:c:h" opt; do
     c)
         coverage_report_file="${OPTARG}"
         ;;
+    e)
+        SF_ENV="${OPTARG}"
+        ;;
     h)
         help 0
         ;;
@@ -91,12 +100,13 @@ merge_gate)
     affected_targets_file="${working_dir}/affected_targets"
     ./bazel/get_affected_targets.sh -b "${bazel}" -f "${affected_targets_file}"
 
-    tag_filter="--test_tag_filters=-autogen"
-
-    query_expr='kind(".*_test rule", rdeps(//... - set('"$(<ci/skip_merge_gate_targets)"') - set('"$(<ci/skip_continuous_run_targets)"'), set('$(<"${affected_targets_file}")')))'
+    query_expr='kind(".*_test rule", rdeps(//... - set('"$(<"ci/targets/quarantine/${SF_ENV}.txt")"') - set('"$(<"ci/targets/slow.txt")"') - set('"$(<"ci/targets/local_only.txt")"'), set('"$(<"${affected_targets_file}")"')))'
     ;;
 continuous_run)
-    query_expr='kind(".*_test rule", //... - set('"$(<ci/skip_continuous_run_targets)"'))'
+    query_expr='kind(".*_test rule", //... - set('"$(<"ci/targets/quarantine/${SF_ENV}.txt")"') - set('"$(<"ci/targets/local_only.txt")"'))'
+    ;;
+quarantined)
+    query_expr='kind(".*_test rule", set('"$(<"ci/targets/quarantine/${SF_ENV}.txt")"') - set('"$(<"ci/targets/local_only.txt")"'))'
     ;;
 local_unittest)
     cache_test_results="--cache_test_results=yes"

ci/build_and_run_tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Usage
-# build_and_run_tests.sh <workspace> [-b <bazel path>] [--env pip|conda] [--mode merge_gate|continuous_run|release] [--with-snowpark] [--report <report_path>]
+# build_and_run_tests.sh <workspace> [-b <bazel path>] [--env pip|conda] [--mode merge_gate|continuous_run] [--with-snowpark] [--report <report_path>]
 #
 # Args
 # workspace: path to the workspace, SnowML code should be in snowml directory.
@@ -11,9 +11,10 @@
 # env: Set the environment, choose from pip and conda
 # mode: Set the tests set to be run.
 #   merge_gate: run affected tests only.
-#   continuous_run (default): run all tests except auto-generated tests. (For nightly run.)
-#   release: run all tests including auto-generated tests. (For releasing)
+#   continuous_run (default): run all tests. (For nightly run. Alias: release)
+#   quarantined: run all quarantined tests.
 # with-snowpark: Build and test with snowpark in snowpark-python directory in the workspace.
+# snowflake-env: The environment of the snowflake, use to determine the test quarantine list
 # report: Path to xml test report
 #
 # Action
@@ -28,7 +29,7 @@ PROG=$0
 
 help() {
     local exit_code=$1
-    echo "Usage: ${PROG} <workspace> [-b <bazel path>] [--env pip|conda] [--mode merge_gate|continuous_run|release] [--with-snowpark] [--report <report_path>]"
+    echo "Usage: ${PROG} <workspace> [-b <bazel path>] [--env pip|conda] [--mode merge_gate|continuous_run|quarantined] [--with-snowpark] [--snowflake-env <sf_env>] [--report <report_path>]"
     exit "${exit_code}"
 }
 
@@ -43,6 +44,7 @@ SNOWML_DIR="snowml"
 SNOWPARK_DIR="snowpark-python"
 IS_NT=false
 JUNIT_REPORT_PATH=""
+SF_ENV="prod3"
 
 while (($#)); do
     case $1 in
@@ -63,12 +65,19 @@ while (($#)); do
         ;;
     --mode)
         shift
-        if [[ $1 = "merge_gate" || $1 = "continuous_run" || $1 = "release" ]]; then
+        if [[ $1 = "merge_gate" || $1 = "continuous_run" || $1 = "quarantined" || $1 = "release" ]]; then
             MODE=$1
+            if [[ $MODE = "release" ]]; then
+                MODE="continuous_run"
+            fi
         else
             help 1
         fi
         ;;
+    --snowflake-env)
+        shift
+        SF_ENV=$1
+        ;;
     --report)
         shift
         JUNIT_REPORT_PATH=$1
@@ -186,20 +195,10 @@ while IFS='' read -r line; do OPTIONAL_REQUIREMENTS+=("$line"); done < <("${_YQ_
 
 # Compare test required dependencies with wheel pkg dependencies and exclude tests if necessary
 EXCLUDE_TESTS=$(mktemp "${TEMP_TEST_DIR}/exclude_tests_XXXXX")
-if [[ ${MODE} = "continuous_run" || ${MODE} = "release" ]]; then
-    ./ci/get_excluded_tests.sh -f "${EXCLUDE_TESTS}" -m unused -b "${BAZEL}"
-elif [[ ${MODE} = "merge_gate" ]]; then
-    ./ci/get_excluded_tests.sh -f "${EXCLUDE_TESTS}" -m all -b "${BAZEL}"
-fi
+./ci/get_excluded_tests.sh -f "${EXCLUDE_TESTS}" -m "${MODE}" -b "${BAZEL}" -e "${SF_ENV}"
 
 # Generate and copy auto-gen tests.
-if [[ ${MODE} = "release" ]]; then
-# When release, we build all autogen tests
-    "${BAZEL}" "${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]+"${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]}"}" build "${BAZEL_ADDITIONAL_BUILD_FLAGS[@]+"${BAZEL_ADDITIONAL_BUILD_FLAGS[@]}"}" //tests/integ/...
-else
-# In other cases, we build required utility only.
-    "${BAZEL}" "${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]+"${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]}"}" build --build_tag_filters=-autogen_build,-autogen "${BAZEL_ADDITIONAL_BUILD_FLAGS[@]+"${BAZEL_ADDITIONAL_BUILD_FLAGS[@]}"}" //tests/integ/...
-fi
+"${BAZEL}" "${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]+"${BAZEL_ADDITIONAL_STARTUP_FLAGS[@]}"}" build "${BAZEL_ADDITIONAL_BUILD_FLAGS[@]+"${BAZEL_ADDITIONAL_BUILD_FLAGS[@]}"}" //tests/integ/...
 
 # Rsync cannot work well with path that has drive letter in Windows,
 # Thus, these two rsync has to use relative path instead of absolute ones.
@@ -330,7 +329,7 @@ echo "Done running ${PROG}"
 #   0: Success;
 #   5: no tests found
 # See https://docs.pytest.org/en/7.1.x/reference/exit-codes.html
-if [[ ${MODE} = "merge_gate" && ${TEST_RETCODE} -eq 5 ]]; then
+if [[ (${MODE} = "merge_gate" || ${MODE} = "quarantined") && ${TEST_RETCODE} -eq 5 ]]; then
     exit 0
 fi
 exit ${TEST_RETCODE}

ci/conda_recipe/meta.yaml

@@ -17,7 +17,7 @@ build:
   noarch: python
 package:
   name: snowflake-ml-python
-  version: 1.2.1
+  version: 1.2.2
 requirements:
   build:
     - python

ci/get_excluded_tests.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 
 # Usage
-# exclude_tests.sh [-b <bazel_path>] [-f <output_path>] [-m unused|unaffected|all]
+# exclude_tests.sh [-b <bazel_path>] [-f <output_path>] [- merge_gate|continuous_run|release]
 #
 # Flags
 # -b: specify path to bazel
 # -f: specify output file path
 # -m: specify the mode from the following options
-#       unused: exclude integration tests whose dependency is not part of the wheel package.
-#               The missing dependency could happen when a new operator is being developed,
-#               but not yet released.
-#       unaffected: exclude integration tests whose dependency is not part of the affected targets
+#       merge_gate: exclude local_only + integration tests whose dependency is not part of the affected targets
 #                   compare to the the merge base to main of current revision.
-#       all (default): exclude the union of above rules.
+#       continuous_run (default): exclude integration tests whose dependency is not part of the wheel package.
+#               The missing dependency could happen when a new operator is being developed,
+#               but not yet released. (Alias: release)
+#       quarantined: exclude all tests that are not quarantined
 #
 
 set -o pipefail
@@ -22,17 +22,18 @@ PROG=$0
 
 help() {
     local exit_code=$1
-    echo "Usage: ${PROG} [-b <bazel_path>] [-f <output_path>] [-m unused|unaffected|all]"
+    echo "Usage: ${PROG} [-b <bazel_path>] [-f <output_path>] [-m merge_gate|continuous_run|release|quarantined]"
     exit "${exit_code}"
 }
 
 echo "Running ${PROG}"
 
 bazel="bazel"
 output_path="/tmp/files_to_exclude"
-mode="all"
+mode="continuous_run"
+SF_ENV="prod3"
 
-while getopts "b:f:m:h" opt; do
+while getopts "b:f:m:e:h" opt; do
     case "${opt}" in
     b)
         bazel=${OPTARG}
@@ -42,9 +43,15 @@ while getopts "b:f:m:h" opt; do
         ;;
     m)
         mode=${OPTARG}
-        if ! [[ $mode = "unused" || $mode = "unaffected" || $mode = "all" ]]; then
+        if ! [[ $mode = "merge_gate" || $mode = "continuous_run" || $mode = "release" || $mode = "quarantined" ]]; then
             help 1
         fi
+        if [[ $mode = "release" ]]; then
+            mode="continuous_run"
+        fi
+        ;;
+    e)
+        SF_ENV=${OPTARG}
         ;;
     h)
         help 0
@@ -61,32 +68,32 @@ done
 working_dir=$(mktemp -d "/tmp/tmp_XXXXX")
 trap 'rm -rf "${working_dir}"' EXIT
 
-if [[ $mode = "unused" || $mode = "all" ]]; then
-    # Compute missing dependencies by subtracting deps included in wheel from deps required by tests.
-    # We only care about dependencies in //snowflake since that's our dev directory.
-    # Reverse search on testing files depending on missing deps and exclude those.
 
-    unused_test_rule_file=${working_dir}/unused_test_rule
+# Compute missing dependencies by subtracting deps included in wheel from deps required by tests.
+# We only care about dependencies in //snowflake since that's our dev directory.
+# Reverse search on testing files depending on missing deps and exclude those.
 
-    # -- Begin of Query Rules Heredoc --
-    cat >"${unused_test_rule_file}" <<EndOfMessage
-    let missing_deps = filter('//snowflake[:/].*', kind('py_library rule', deps(tests/...) except deps(wheel))) in
-        labels(srcs, kind('py_test rule', rdeps(//tests/..., \$missing_deps, 1)))
+unused_test_rule_file=${working_dir}/unused_test_rule
+
+# -- Begin of Query Rules Heredoc --
+cat >"${unused_test_rule_file}" <<EndOfMessage
+let missing_deps = filter('//snowflake[:/].*', kind('py_library rule', deps(tests/...) except deps(:wheel))) in
+    labels(srcs, kind('py_test rule', rdeps(//tests/..., \$missing_deps, 1) + set($(<"ci/targets/quarantine/${SF_ENV}.txt"))))
 EndOfMessage
-    # -- End of Query Rules Heredoc --
+# -- End of Query Rules Heredoc --
 
-    unused_test_targets=$("${bazel}" query --query_file="${unused_test_rule_file}")
-fi
+unused_test_targets=$("${bazel}" query --query_file="${unused_test_rule_file}")
 
-if [[ $mode = "unaffected" || $mode = "all" ]]; then
+
+if [[ $mode = "merge_gate" ]]; then
     affected_targets_file="${working_dir}/affected_targets"
     ./bazel/get_affected_targets.sh -b "${bazel}" -f "${affected_targets_file}"
 
     unaffected_test_rule_file=${working_dir}/unaffected_test_rule
 
     # -- Begin of Query Rules Heredoc --
     cat >"${unaffected_test_rule_file}" <<EndOfMessage
-    let affected_targets = set($(<"${affected_targets_file}")) - set($(<ci/skip_merge_gate_targets)) in
+    let affected_targets = set($(<"${affected_targets_file}")) - set($(<"ci/targets/slow.txt")) - set($(<"ci/targets/quarantine/${SF_ENV}.txt")) in
         let unaffected_targets = //tests/... - deps(\$affected_targets) - rdeps(//tests/..., \$affected_targets) in
             labels(srcs, \$unaffected_targets)
 EndOfMessage
@@ -98,17 +105,26 @@ fi
 targets_to_exclude_file="${working_dir}/targets_to_exclude_file"
 
 case "${mode}" in
-unused)
+continuous_run)
     echo "${unused_test_targets}" >"${targets_to_exclude_file}"
     ;;
-unaffected)
-    echo "${unaffected_test_targets}" >"${targets_to_exclude_file}"
-    ;;
-all)
+merge_gate)
     # Concat and deduplicate.
     targets_to_exclude=$(printf "%s\n%s\n" "${unused_test_targets}" "${unaffected_test_targets}" | awk '!a[$0]++')
     echo "${targets_to_exclude}" >"${targets_to_exclude_file}"
     ;;
+quarantined)
+    quarantined_test_rule_file=${working_dir}/quarantined_test_rule
+
+# -- Begin of Query Rules Heredoc --
+    cat >"${quarantined_test_rule_file}" <<EndOfMessage
+    labels(srcs, kind('py_test rule', //tests/... - set($(<"ci/targets/quarantine/${SF_ENV}.txt"))))
+EndOfMessage
+# -- End of Query Rules Heredoc --
+
+    quarantined_test_targets=$("${bazel}" query --query_file="${quarantined_test_rule_file}")
+    echo "${quarantined_test_targets}" >"${targets_to_exclude_file}"
+    ;;
 *)
     help 1
     ;;
@@ -118,11 +134,11 @@ excluded_test_source_rule_file=${working_dir}/excluded_test_source_rule
 
 # -- Begin of Query Rules Heredoc --
 cat >"${excluded_test_source_rule_file}" <<EndOfMessage
-let skip_continuous_run_targets = set($(<ci/skip_continuous_run_targets)) in
+let skip_continuous_run_targets = set($(<"ci/targets/local_only.txt")) in
     let targets_to_exclude = set($(<"${targets_to_exclude_file}")) + labels(srcs, \$skip_continuous_run_targets) in
             kind('source file', \$targets_to_exclude)
 EndOfMessage
-# -- End of Query Rules Heredoc --
+    # -- End of Query Rules Heredoc --
 
 ${bazel} query --query_file="${excluded_test_source_rule_file}" \
     --output location |

ci/targets/slow.txt

@@ -0,0 +1,3 @@
+//tests/integ/snowflake/ml/model:deployment_to_snowservice_integ_test
+//tests/integ/snowflake/ml/registry:model_registry_snowservice_integ_test
+//tests/integ/snowflake/ml/model:spcs_llm_model_integ_test