Add support for volumes in deployment bind/unbind commands (#2527)

## Changes
1. Changed `FindResourceByConfigKey` to return volume resources
2. Implemented `Exists` method on Volume resource

## Why
This PR adds support for volume resources in deployment operations,
enabling users to:
- Bind experiments using `databricks bundle deployment bind
<myvolume_key> <myvolume_full_name>`
- Unbind experiments using `databricks bundle deployment unbind
<myvolume_key>`

Where:
- `myvolume_key` is a resource key defined in the bundle's .yml file
- `myvolume_full_name` references an existing volume in the Databricks
workspace using its fully qualified (3-level) name

These capabilities allow for more flexible resource management of
volumes within bundles.

## Tests
Added a new acceptance test that tests bind and unbind methods together
with bundle deployment and destruction.

Author: anton-107

NEXT_CHANGELOG.md

@@ -8,6 +8,7 @@
 
 * Processing 'artifacts' section is now done in "bundle validate" (adding defaults, inferring "build", asserting required fields) ([#2526])(https://github.com/databricks/cli/pull/2526))
 * When uploading artifacts, include relative path in log message ([#2539])(https://github.com/databricks/cli/pull/2539))
-* Add support for clusters in deployment bind/unbind commands ([#2536](https://github.com/databricks/cli/pull/2536))
+* Added support for clusters in deployment bind/unbind commands ([#2536](https://github.com/databricks/cli/pull/2536))
+* Added support for volumes in deployment bind/unbind commands ([#2527](https://github.com/databricks/cli/pull/2527))
 
 ### API Changes

acceptance/acceptance_test.go

@@ -44,7 +44,7 @@ var (
 )
 
 // In order to debug CLI running under acceptance test, search for TestInprocessMode and update
-// the test name there, e..g "bundle/variables/empty".
+// the test name there, e.g. "bundle/variables/empty".
 // Then install your breakpoints and click "debug test" near TestInprocessMode in VSCODE.
 
 // If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs

acceptance/bundle/deployment/bind/volume/databricks.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: bind-dashboard-test-$UNIQUE_NAME
+
+resources:
+  volumes:
+    volume1:
+      name: $VOLUME_NAME
+      schema_name: $SCHEMA_NAME
+      catalog_name: $CATALOG_NAME

acceptance/bundle/deployment/bind/volume/output.txt

@@ -0,0 +1,42 @@
+
+>>> [CLI] schemas create test-schema-[UUID] main
+{
+  "full_name": "main.test-schema-[UUID]",
+  "catalog_name": "main"
+}
+
+=== Create a pre-defined volume: 
+>>> [CLI] bundle deployment bind volume1 main.test-schema-[UUID].volume-[UUID] --auto-approve
+Updating deployment state...
+Successfully bound databricks_volume with an id 'main.test-schema-[UUID].volume-[UUID]'. Run 'bundle deploy' to deploy changes to your workspace
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/bind-dashboard-test-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] volumes read main.test-schema-[UUID].volume-[UUID]
+{
+  "catalog_name": "main",
+  "full_name": "main.test-schema-[UUID].volume-[UUID]",
+  "schema_name": "test-schema-[UUID]",
+  "volume_type": "MANAGED"
+}
+
+>>> [CLI] bundle deployment unbind volume1
+Updating deployment state...
+
+>>> [CLI] bundle destroy --auto-approve
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/bind-dashboard-test-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!
+
+>>> [CLI] volumes read main.test-schema-[UUID].volume-[UUID]
+{
+  "catalog_name": "main",
+  "full_name": "main.test-schema-[UUID].volume-[UUID]",
+  "schema_name": "test-schema-[UUID]",
+  "volume_type": "MANAGED"
+}

acceptance/bundle/deployment/bind/volume/script

@@ -0,0 +1,34 @@
+VOLUME_NAME="volume-$(uuid)"
+SCHEMA_NAME="test-schema-$(uuid)"
+if [ -z "$CLOUD_ENV" ]; then
+    VOLUME_NAME="volume-6260d50f-e8ff-4905-8f28-812345678903"   # use hard-coded uuid when running locally
+    SCHEMA_NAME="test-schema-6260d50f-e8ff-4905-8f28-812345678903"
+fi
+export VOLUME_NAME
+export SCHEMA_NAME
+export CATALOG_NAME="main"
+envsubst < databricks.yml.tmpl > databricks.yml
+
+VOLUME_TYPE="MANAGED"
+
+trace $CLI schemas create "${SCHEMA_NAME}" ${CATALOG_NAME} | jq '{full_name, catalog_name}'
+
+title "Create a pre-defined volume: "
+VOLUME_FULL_NAME=$($CLI volumes create "${CATALOG_NAME}" "${SCHEMA_NAME}" "${VOLUME_NAME}" "${VOLUME_TYPE}" | jq -r '.full_name')
+
+cleanupRemoveVolume() {
+    $CLI volumes delete "${VOLUME_FULL_NAME}"
+}
+trap cleanupRemoveVolume EXIT
+
+trace $CLI bundle deployment bind volume1 "${VOLUME_FULL_NAME}" --auto-approve
+
+trace $CLI bundle deploy
+
+trace $CLI volumes read "${VOLUME_FULL_NAME}" | jq '{catalog_name, full_name, schema_name, volume_type}'
+
+trace $CLI bundle deployment unbind volume1
+
+trace $CLI bundle destroy --auto-approve
+
+trace $CLI volumes read "${VOLUME_FULL_NAME}" | jq '{catalog_name, full_name, schema_name, volume_type}'

acceptance/bundle/deployment/bind/volume/test.toml

@@ -0,0 +1,40 @@
+Local = true
+Cloud = true
+RequiresUnityCatalog = true
+
+Ignore = [
+    "databricks.yml",
+]
+
+[[Server]]
+Pattern = "POST /api/2.1/unity-catalog/schemas"
+Response.Body = '''
+{
+    "name":"test-schema-6260d50f-e8ff-4905-8f28-812345678903",
+    "catalog_name":"main",
+    "full_name":"main.test-schema-6260d50f-e8ff-4905-8f28-812345678903"
+}
+'''
+
+[[Server]]
+Pattern = "POST /api/2.1/unity-catalog/volumes"
+Response.Body = '''
+{
+    "full_name":"main.test-schema-6260d50f-e8ff-4905-8f28-812345678903.volume-6260d50f-e8ff-4905-8f28-812345678903"
+}
+'''
+
+[[Server]]
+Pattern = "GET /api/2.1/unity-catalog/volumes/{volume_fullname}"
+Response.Body = '''
+{
+  "catalog_name": "main",
+  "schema_name": "test-schema-6260d50f-e8ff-4905-8f28-812345678903",
+  "name": "volume-6260d50f-e8ff-4905-8f28-812345678903",
+  "full_name": "main.test-schema-6260d50f-e8ff-4905-8f28-812345678903.volume-6260d50f-e8ff-4905-8f28-812345678903",
+  "volume_type": "MANAGED"
+}
+'''
+
+[[Server]]
+Pattern = "DELETE /api/2.1/unity-catalog/volumes/{volume_fullname}"

bundle/config/resources.go

@@ -130,6 +130,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
 		}
 	}
 
+	for k := range r.Volumes {
+		if k == key {
+			found = append(found, r.Volumes[k])
+		}
+	}
+
 	if len(found) == 0 {
 		return nil, fmt.Errorf("no such resource: %s", key)
 	}

bundle/config/resources/volume.go

@@ -6,6 +6,9 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go/apierr"
+
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/marshal"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
@@ -33,8 +36,24 @@ func (v Volume) MarshalJSON() ([]byte, error) {
 	return marshal.Marshal(v)
 }
 
-func (v *Volume) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
-	return false, errors.New("volume.Exists() is not supported")
+func (v *Volume) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullyQualifiedName string) (bool, error) {
+	_, err := w.Volumes.Read(ctx, catalog.ReadVolumeRequest{
+		Name: fullyQualifiedName,
+	})
+	if err != nil {
+		log.Debugf(ctx, "volume with fully qualified name %s does not exist: %v", fullyQualifiedName, err)
+
+		var aerr *apierr.APIError
+		if errors.As(err, &aerr) {
+			if aerr.StatusCode == 404 {
+				return false, nil
+			}
+		}
+
+		return false, err
+	}
+
+	return true, nil
 }
 
 func (v *Volume) TerraformResourceName() string {

bundle/config/resources/volume_test.go

@@ -0,0 +1,26 @@
+package resources
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVolumeNotFound(t *testing.T) {
+	ctx := context.Background()
+
+	m := mocks.NewMockWorkspaceClient(t)
+	m.GetMockVolumesAPI().On("Read", mock.Anything, mock.Anything).Return(nil, &apierr.APIError{
+		StatusCode: 404,
+	})
+
+	s := &Volume{}
+	exists, err := s.Exists(ctx, m.WorkspaceClient, "non-existent-volume")
+
+	require.Falsef(t, exists, "Exists should return false when getting a 404 response from Workspace")
+	require.NoErrorf(t, err, "Exists should not return an error when getting a 404 response from Workspace")
+}