Dev/paul/8963 upgrade azblob

.gitignore

@@ -1,3 +1,6 @@
 .task/
 .local/
 .vscode/
+.bash_history
+.viminfo
+

azblob/accessconditions.go

@@ -3,33 +3,38 @@ package azblob
 import (
 	"errors"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
 )
 
-func storerOptionConditions(options *StorerOptions) (azStorageBlob.BlobAccessConditions, error) {
+func storerOptionConditions(options *StorerOptions) (blob.AccessConditions, error) {
 
-	var blobAccessConditions azStorageBlob.BlobAccessConditions
+	var blobAccessConditions blob.AccessConditions
 	if options.leaseID == "" && options.etagCondition == EtagNotUsed {
 		return blobAccessConditions, nil
 	}
 	if options.etag == "" && options.etagCondition != EtagNotUsed {
 		return blobAccessConditions, errors.New("etag value missing")
 	}
 
-	blobAccessConditions = azStorageBlob.BlobAccessConditions{}
+	blobAccessConditions = blob.AccessConditions{}
 	if options.leaseID != "" {
-		blobAccessConditions.LeaseAccessConditions = &azStorageBlob.LeaseAccessConditions{
+		blobAccessConditions.LeaseAccessConditions = &lease.AccessConditions{
 			LeaseID: &options.leaseID,
 		}
 	}
 
-	blobAccessConditions.ModifiedAccessConditions = &azStorageBlob.ModifiedAccessConditions{}
+	blobAccessConditions.ModifiedAccessConditions = &blob.ModifiedAccessConditions{}
 
 	switch options.etagCondition {
 	case ETagMatch:
-		blobAccessConditions.ModifiedAccessConditions.IfMatch = &options.etag
+		t := azcore.ETag(options.etag)
+		blobAccessConditions.ModifiedAccessConditions.IfMatch = &t
 	case ETagNoneMatch:
-		blobAccessConditions.ModifiedAccessConditions.IfNoneMatch = &options.etag
+		t := azcore.ETag(options.etag)
+		blobAccessConditions.ModifiedAccessConditions.IfNoneMatch = &t
 	case TagsWhere:
 		blobAccessConditions.ModifiedAccessConditions.IfTags = &options.etag
 	default:

azblob/delete.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	msazblob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/datatrails/go-datatrails-common/logger"
 )
 
@@ -15,16 +17,12 @@ func (azp *Storer) Delete(
 ) error {
 	logger.Sugar.Debugf("Delete blob %s", identity)
 
-	blockBlobClient, err := azp.containerClient.NewBlockBlobClient(identity)
-	if err != nil {
-		logger.Sugar.Infof("Cannot get block blob client blob: %v", err)
-		return ErrorFromError(err)
-	}
+	blockBlobClient := azp.containerClient.NewBlockBlobClient(identity)
 
-	_, err = blockBlobClient.Delete(ctx, nil)
-	var terr *msazblob.StorageError
+	_, err := blockBlobClient.Delete(ctx, nil)
+	var terr *azcore.ResponseError
 	if errors.As(err, &terr) {
-		resp := terr.Response()
+		resp := terr.RawResponse
 		if resp.Body != nil {
 			defer resp.Body.Close()
 		}

azblob/download.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 
 	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 
 	"github.com/datatrails/go-datatrails-common/logger"
 )
@@ -31,11 +32,7 @@ func (azp *Storer) getTags(
 	var err error
 	logger.Sugar.Debugf("getTags BlockBlob URL %s", identity)
 
-	blobClient, err := azp.containerClient.NewBlobClient(identity)
-	if err != nil {
-		logger.Sugar.Debugf("getTags BlockBlob Client %s error: %v", identity, err)
-		return nil, ErrorFromError(err)
-	}
+	blobClient := azp.containerClient.NewBlobClient(identity)
 
 	resp, err := blobClient.GetTags(ctx, nil)
 	if err != nil {
@@ -55,13 +52,11 @@ func (azp *Storer) getTags(
 func (azp *Storer) getMetadata(
 	ctx context.Context,
 	identity string,
-) (map[string]string, error) {
+) (map[string]*string, error) {
 	logger.Sugar.Debugf("getMetadata BlockBlob URL %s", identity)
 
-	blobClient, err := azp.containerClient.NewBlobClient(identity)
-	if err != nil {
-		return nil, ErrorFromError(err)
-	}
+	blobClient := azp.containerClient.NewBlobClient(identity)
+
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	resp, err := blobClient.GetProperties(ctx, nil)
@@ -143,16 +138,14 @@ func (azp *Storer) Reader(
 	}
 
 	logger.Sugar.Debugf("Creating New io.Reader")
-	resp.BlobClient, err = azp.containerClient.NewBlobClient(identity)
-	if err != nil {
-		return nil, ErrorFromError(err)
-	}
-	countToEnd := int64(azStorageBlob.CountToEnd)
-	get, err := resp.BlobClient.Download(
+	resp.BlobClient = azp.containerClient.NewBlobClient(identity)
+	get, err := resp.BlobClient.DownloadStream(
 		ctx,
-		&azStorageBlob.BlobDownloadOptions{
-			BlobAccessConditions: &blobAccessConditions,
-			Count:                &countToEnd,
+		&blob.DownloadStreamOptions{
+			AccessConditions: &blobAccessConditions,
+			Range: blob.HTTPRange{
+				Count: int64(blob.CountToEnd),
+			},
 		},
 	)
 
@@ -173,10 +166,7 @@ func (azp *Storer) Reader(
 		if options.getMetadata == BothMetadataAndBlob {
 			_ = readerResponseMetadata(resp, resp.Metadata) // the parse error is benign
 		}
-	}
-
-	if get.RawResponse != nil {
-		resp.Reader = get.Body(nil)
+		resp.Reader = get.Body
 	}
 	return resp, err
 }

azblob/error.go

@@ -6,7 +6,10 @@ import (
 	"errors"
 	"net/http"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"github.com/datatrails/go-datatrails-common/logger"
 )
 
@@ -43,9 +46,9 @@ func (e *Error) Unwrap() error {
 // StatusCode returns status code for failing request or 500 if code is not available on the error
 func (e *Error) StatusCode() int {
 
-	var terr *azStorageBlob.StorageError
+	var terr *azcore.ResponseError
 	if errors.As(e.err, &terr) {
-		resp := terr.Response()
+		resp := terr.RawResponse
 		if resp.Body != nil {
 			defer resp.Body.Close()
 		}
@@ -62,7 +65,7 @@ func (e *Error) StatusCode() int {
 
 // StorageErrorCode returns the underlying azure storage ErrorCode string eg "BlobNotFound"
 func (e *Error) StorageErrorCode() string {
-	var terr *azStorageBlob.StorageError
+	var terr *azcore.ResponseError
 	if errors.As(e.err, &terr) {
 		if terr.ErrorCode != "" {
 			return string(terr.ErrorCode)
@@ -74,5 +77,5 @@ func (e *Error) StorageErrorCode() string {
 // IsConditionNotMet returns true if the err is the storage code indicating that
 // a If- header predicate (eg ETag) was not met
 func (e *Error) IsConditionNotMet() bool {
-	return e.StorageErrorCode() == string(azStorageBlob.StorageErrorCodeConditionNotMet)
+	return bloberror.HasCode(e, bloberror.ConditionNotMet)
 }

azblob/lease.go

@@ -9,7 +9,10 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
 
 	"github.com/datatrails/go-datatrails-common/logger"
 )
@@ -83,25 +86,24 @@ func (azp *Storer) AcquireLeaseRenewable(
 func (azp *Storer) acquireLease(
 	ctx context.Context, objectname string, leaseTimeout int32,
 ) (
-	*azStorageBlob.BlobAcquireLeaseResponse, *azStorageBlob.BlobLeaseClient, error,
+	*lease.ContainerAcquireResponse, *lease.ContainerClient, error,
 ) {
 	logger.Sugar.Debugf("acquireLease: %v", objectname)
 
-	blockBlobClient, err := azp.containerClient.NewBlockBlobClient(objectname)
+	leaseBlobClient, err := lease.NewContainerClient(
+		azp.containerClient,
+		&lease.ContainerClientOptions{
+			LeaseID: to.Ptr(objectname),
+		},
+	)
 	if err != nil {
 		logger.Sugar.Infof("cannot create block blob client %s: %v", objectname, err)
 		return nil, nil, err
 	}
-	leaseBlobClient, err := blockBlobClient.NewBlobLeaseClient(nil)
-	if err != nil {
-		logger.Sugar.Infof("cannot create lease Blob %s: %v", objectname, err)
-		return nil, nil, err
-	}
 	lease, err := leaseBlobClient.AcquireLease(
 		ctx,
-		&azStorageBlob.BlobAcquireLeaseOptions{
-			Duration: &leaseTimeout,
-		},
+		leaseTimeout,
+		nil,
 	)
 
 	return &lease, leaseBlobClient, err
@@ -120,12 +122,12 @@ func (azp *Storer) ReleaseLeaseDeferable(ctx context.Context, objectname string,
 func (azp *Storer) ReleaseLease(ctx context.Context, objectname string, leaseID string,
 ) error {
 	logger.Sugar.Debugf("ReleaseLease: %v", objectname)
-	blockBlobClient, err := azp.containerClient.NewBlockBlobClient(objectname)
-	if err != nil {
-		logger.Sugar.Infof("cannot create block Blob client %s: %v", objectname, err)
-		return err
-	}
-	leaseBlobClient, err := blockBlobClient.NewBlobLeaseClient(&leaseID)
+	leaseBlobClient, err := lease.NewContainerClient(
+		azp.containerClient,
+		&lease.ContainerClientOptions{
+			LeaseID: to.Ptr(objectname),
+		},
+	)
 	if err != nil {
 		logger.Sugar.Infof("cannot create lease Blob %s: %v", objectname, err)
 		return err

azblob/list.go

@@ -6,6 +6,9 @@ import (
 	"context"
 
 	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 
 	"github.com/datatrails/go-datatrails-common/logger"
 )
@@ -27,23 +30,22 @@ func (azp *Storer) Count(ctx context.Context, tagsFilter string) (int64, error)
 //
 // tagsFilter example: "dog='germanshepherd' and penguin='emperorpenguin'"
 // Returns all blobs with the specific tag filter
-func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string) ([]*azStorageBlob.FilterBlobItem, error) {
+func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string) ([]*service.FilterBlobItem, error) {
 	logger.Sugar.Debugf("FilteredList")
 
-	var filteredBlobs []*azStorageBlob.FilterBlobItem
+	var filteredBlobs []*service.FilterBlobItem
 	var err error
 
-	result, err := azp.serviceClient.FindBlobsByTags(
+	result, err := azp.serviceClient.FilterBlobs(
 		ctx,
-		&azStorageBlob.ServiceFilterBlobsOptions{
-			Where: &tagsFilter,
-		},
+		tagsFilter,
+		nil,
 	)
 	if err != nil {
-		return filteredBlobs, err
+		return nil, err
 	}
 
-	filteredBlobs = result.Blobs
+	filteredBlobs = result.FilterBlobSegment.Blobs
 
 	return filteredBlobs, err
 }
@@ -52,11 +54,7 @@ type ListerResponse struct {
 	Marker ListMarker // nil if no more pages
 	Prefix string
 
-	// Standard request status things
-	StatusCode int // For If- header fails, err can be nil and code can be 304
-	Status     string
-
-	Items []*azStorageBlob.BlobItemInternal
+	Items []*container.BlobItem
 }
 
 func (azp *Storer) List(ctx context.Context, opts ...Option) (*ListerResponse, error) {
@@ -65,38 +63,45 @@ func (azp *Storer) List(ctx context.Context, opts ...Option) (*ListerResponse, e
 	for _, opt := range opts {
 		opt(options)
 	}
-	o := azStorageBlob.ContainerListBlobsFlatOptions{
+	o := azStorageBlob.ListBlobsFlatOptions{
 		Marker: options.listMarker,
+		Include: container.ListBlobsInclude{
+			Metadata: options.listIncludeMetadata,
+			Tags:     options.listIncludeTags,
+		},
 	}
 	if options.listPrefix != "" {
 		o.Prefix = &options.listPrefix
 	}
-	if options.listIncludeTags {
-		o.Include = append(o.Include, azStorageBlob.ListBlobsIncludeItemTags)
-	}
-	if options.listIncludeMetadata {
-		o.Include = append(o.Include, azStorageBlob.ListBlobsIncludeItemMetadata)
-	}
 
 	// TODO: v1.21 feature which would be great
 	// if options.listDelim != "" {
 	// }
-	r := &ListerResponse{}
-	pager := azp.containerClient.ListBlobsFlat(&o)
-	if !pager.NextPage(ctx) {
-		return r, nil
-	}
-	resp := pager.PageResponse()
-	r.Status = resp.RawResponse.Status
-	r.StatusCode = resp.RawResponse.StatusCode
+	r := &ListerResponse{Items: []*container.BlobItem{}}
 
+	// blob listings are returned across multiple pages
+	pager := azp.containerClient.NewListBlobsFlatPager(&o)
+	resp, err := pager.NextPage(ctx)
+	if err != nil {
+		return nil, err
+	}
 	if resp.Prefix != nil {
 		r.Prefix = *resp.Prefix
 	}
-
-	// Note: we pass on the azure type otherwise we would be copying for no good
-	// reason. let the caller decided how to deal with that
-	r.Items = resp.Segment.BlobItems
-
+	r.Items = append(r.Items, resp.Segment.BlobItems...)
+	// continue fetching pages until no more remain
+	for pager.More() {
+		// advance to the next page
+		resp, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, err
+		}
+		if resp.Prefix != nil {
+			r.Prefix = *resp.Prefix
+		}
+		// Note: we pass on the azure type otherwise we would be copying for no good
+		// reason. let the caller decided how to deal with that
+		r.Items = append(r.Items, resp.Segment.BlobItems...)
+	}
 	return r, nil
 }

azblob/options.go

@@ -34,7 +34,7 @@ func (g GetMetadata) String() string {
 // StorerOptions - optional args for specifying optional behaviour
 type StorerOptions struct {
 	leaseID        string
-	metadata       map[string]string
+	metadata       map[string]*string
 	tags           map[string]string
 	getMetadata    GetMetadata
 	getTags        bool
@@ -150,7 +150,7 @@ func WithLeaseID(leaseID string) Option {
 }
 
 // WithMetadata specifies metadata to add - Write() only
-func WithMetadata(metadata map[string]string) Option {
+func WithMetadata(metadata map[string]*string) Option {
 	return func(a *StorerOptions) {
 		a.metadata = metadata
 	}