diff --git a/.gitignore b/.gitignore index f5f294c..7db19df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ .task/ .local/ .vscode/ +.bash_history +.viminfo + diff --git a/azblob/accessconditions.go b/azblob/accessconditions.go index ff35350..47fc777 100644 --- a/azblob/accessconditions.go +++ b/azblob/accessconditions.go @@ -3,12 +3,15 @@ package azblob import ( "errors" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" ) -func storerOptionConditions(options *StorerOptions) (azStorageBlob.BlobAccessConditions, error) { +func storerOptionConditions(options *StorerOptions) (blob.AccessConditions, error) { - var blobAccessConditions azStorageBlob.BlobAccessConditions + var blobAccessConditions blob.AccessConditions if options.leaseID == "" && options.etagCondition == EtagNotUsed { return blobAccessConditions, nil } @@ -16,20 +19,22 @@ func storerOptionConditions(options *StorerOptions) (azStorageBlob.BlobAccessCon return blobAccessConditions, errors.New("etag value missing") } - blobAccessConditions = azStorageBlob.BlobAccessConditions{} + blobAccessConditions = blob.AccessConditions{} if options.leaseID != "" { - blobAccessConditions.LeaseAccessConditions = &azStorageBlob.LeaseAccessConditions{ + blobAccessConditions.LeaseAccessConditions = &lease.AccessConditions{ LeaseID: &options.leaseID, } } - blobAccessConditions.ModifiedAccessConditions = &azStorageBlob.ModifiedAccessConditions{} + blobAccessConditions.ModifiedAccessConditions = &blob.ModifiedAccessConditions{} switch options.etagCondition { case ETagMatch: - blobAccessConditions.ModifiedAccessConditions.IfMatch = &options.etag + t := azcore.ETag(options.etag) + blobAccessConditions.ModifiedAccessConditions.IfMatch = &t case ETagNoneMatch: - blobAccessConditions.ModifiedAccessConditions.IfNoneMatch = &options.etag + t := azcore.ETag(options.etag) + blobAccessConditions.ModifiedAccessConditions.IfNoneMatch = &t case TagsWhere: blobAccessConditions.ModifiedAccessConditions.IfTags = &options.etag default: diff --git a/azblob/delete.go b/azblob/delete.go index 23d76b9..5639385 100644 --- a/azblob/delete.go +++ b/azblob/delete.go @@ -4,7 +4,9 @@ import ( "context" "errors" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" msazblob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" "github.com/datatrails/go-datatrails-common/logger" ) @@ -15,16 +17,12 @@ func (azp *Storer) Delete( ) error { logger.Sugar.Debugf("Delete blob %s", identity) - blockBlobClient, err := azp.containerClient.NewBlockBlobClient(identity) - if err != nil { - logger.Sugar.Infof("Cannot get block blob client blob: %v", err) - return ErrorFromError(err) - } + blockBlobClient := azp.containerClient.NewBlockBlobClient(identity) - _, err = blockBlobClient.Delete(ctx, nil) - var terr *msazblob.StorageError + _, err := blockBlobClient.Delete(ctx, nil) + var terr *azcore.ResponseError if errors.As(err, &terr) { - resp := terr.Response() + resp := terr.RawResponse if resp.Body != nil { defer resp.Body.Close() } diff --git a/azblob/download.go b/azblob/download.go index 5e814cc..516b069 100644 --- a/azblob/download.go +++ b/azblob/download.go @@ -9,6 +9,7 @@ import ( "net/http" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/datatrails/go-datatrails-common/logger" ) @@ -31,11 +32,7 @@ func (azp *Storer) getTags( var err error logger.Sugar.Debugf("getTags BlockBlob URL %s", identity) - blobClient, err := azp.containerClient.NewBlobClient(identity) - if err != nil { - logger.Sugar.Debugf("getTags BlockBlob Client %s error: %v", identity, err) - return nil, ErrorFromError(err) - } + blobClient := azp.containerClient.NewBlobClient(identity) resp, err := blobClient.GetTags(ctx, nil) if err != nil { @@ -55,13 +52,11 @@ func (azp *Storer) getTags( func (azp *Storer) getMetadata( ctx context.Context, identity string, -) (map[string]string, error) { +) (map[string]*string, error) { logger.Sugar.Debugf("getMetadata BlockBlob URL %s", identity) - blobClient, err := azp.containerClient.NewBlobClient(identity) - if err != nil { - return nil, ErrorFromError(err) - } + blobClient := azp.containerClient.NewBlobClient(identity) + ctx, cancel := context.WithCancel(ctx) defer cancel() resp, err := blobClient.GetProperties(ctx, nil) @@ -143,16 +138,14 @@ func (azp *Storer) Reader( } logger.Sugar.Debugf("Creating New io.Reader") - resp.BlobClient, err = azp.containerClient.NewBlobClient(identity) - if err != nil { - return nil, ErrorFromError(err) - } - countToEnd := int64(azStorageBlob.CountToEnd) - get, err := resp.BlobClient.Download( + resp.BlobClient = azp.containerClient.NewBlobClient(identity) + get, err := resp.BlobClient.DownloadStream( ctx, - &azStorageBlob.BlobDownloadOptions{ - BlobAccessConditions: &blobAccessConditions, - Count: &countToEnd, + &blob.DownloadStreamOptions{ + AccessConditions: &blobAccessConditions, + Range: blob.HTTPRange{ + Count: int64(blob.CountToEnd), + }, }, ) @@ -173,10 +166,7 @@ func (azp *Storer) Reader( if options.getMetadata == BothMetadataAndBlob { _ = readerResponseMetadata(resp, resp.Metadata) // the parse error is benign } - } - - if get.RawResponse != nil { - resp.Reader = get.Body(nil) + resp.Reader = get.Body } return resp, err } diff --git a/azblob/error.go b/azblob/error.go index ace762f..761f75d 100644 --- a/azblob/error.go +++ b/azblob/error.go @@ -6,7 +6,10 @@ import ( "errors" "net/http" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/datatrails/go-datatrails-common/logger" ) @@ -43,9 +46,9 @@ func (e *Error) Unwrap() error { // StatusCode returns status code for failing request or 500 if code is not available on the error func (e *Error) StatusCode() int { - var terr *azStorageBlob.StorageError + var terr *azcore.ResponseError if errors.As(e.err, &terr) { - resp := terr.Response() + resp := terr.RawResponse if resp.Body != nil { defer resp.Body.Close() } @@ -62,7 +65,7 @@ func (e *Error) StatusCode() int { // StorageErrorCode returns the underlying azure storage ErrorCode string eg "BlobNotFound" func (e *Error) StorageErrorCode() string { - var terr *azStorageBlob.StorageError + var terr *azcore.ResponseError if errors.As(e.err, &terr) { if terr.ErrorCode != "" { return string(terr.ErrorCode) @@ -74,5 +77,5 @@ func (e *Error) StorageErrorCode() string { // IsConditionNotMet returns true if the err is the storage code indicating that // a If- header predicate (eg ETag) was not met func (e *Error) IsConditionNotMet() bool { - return e.StorageErrorCode() == string(azStorageBlob.StorageErrorCodeConditionNotMet) + return bloberror.HasCode(e, bloberror.ConditionNotMet) } diff --git a/azblob/lease.go b/azblob/lease.go index af3507f..d635e40 100644 --- a/azblob/lease.go +++ b/azblob/lease.go @@ -9,7 +9,10 @@ import ( "net/http" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" "github.com/datatrails/go-datatrails-common/logger" ) @@ -83,25 +86,24 @@ func (azp *Storer) AcquireLeaseRenewable( func (azp *Storer) acquireLease( ctx context.Context, objectname string, leaseTimeout int32, ) ( - *azStorageBlob.BlobAcquireLeaseResponse, *azStorageBlob.BlobLeaseClient, error, + *lease.ContainerAcquireResponse, *lease.ContainerClient, error, ) { logger.Sugar.Debugf("acquireLease: %v", objectname) - blockBlobClient, err := azp.containerClient.NewBlockBlobClient(objectname) + leaseBlobClient, err := lease.NewContainerClient( + azp.containerClient, + &lease.ContainerClientOptions{ + LeaseID: to.Ptr(objectname), + }, + ) if err != nil { logger.Sugar.Infof("cannot create block blob client %s: %v", objectname, err) return nil, nil, err } - leaseBlobClient, err := blockBlobClient.NewBlobLeaseClient(nil) - if err != nil { - logger.Sugar.Infof("cannot create lease Blob %s: %v", objectname, err) - return nil, nil, err - } lease, err := leaseBlobClient.AcquireLease( ctx, - &azStorageBlob.BlobAcquireLeaseOptions{ - Duration: &leaseTimeout, - }, + leaseTimeout, + nil, ) return &lease, leaseBlobClient, err @@ -120,12 +122,12 @@ func (azp *Storer) ReleaseLeaseDeferable(ctx context.Context, objectname string, func (azp *Storer) ReleaseLease(ctx context.Context, objectname string, leaseID string, ) error { logger.Sugar.Debugf("ReleaseLease: %v", objectname) - blockBlobClient, err := azp.containerClient.NewBlockBlobClient(objectname) - if err != nil { - logger.Sugar.Infof("cannot create block Blob client %s: %v", objectname, err) - return err - } - leaseBlobClient, err := blockBlobClient.NewBlobLeaseClient(&leaseID) + leaseBlobClient, err := lease.NewContainerClient( + azp.containerClient, + &lease.ContainerClientOptions{ + LeaseID: to.Ptr(objectname), + }, + ) if err != nil { logger.Sugar.Infof("cannot create lease Blob %s: %v", objectname, err) return err diff --git a/azblob/list.go b/azblob/list.go index 2107836..c5194ab 100644 --- a/azblob/list.go +++ b/azblob/list.go @@ -6,6 +6,9 @@ import ( "context" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" "github.com/datatrails/go-datatrails-common/logger" ) @@ -27,23 +30,22 @@ func (azp *Storer) Count(ctx context.Context, tagsFilter string) (int64, error) // // tagsFilter example: "dog='germanshepherd' and penguin='emperorpenguin'" // Returns all blobs with the specific tag filter -func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string) ([]*azStorageBlob.FilterBlobItem, error) { +func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string) ([]*service.FilterBlobItem, error) { logger.Sugar.Debugf("FilteredList") - var filteredBlobs []*azStorageBlob.FilterBlobItem + var filteredBlobs []*service.FilterBlobItem var err error - result, err := azp.serviceClient.FindBlobsByTags( + result, err := azp.serviceClient.FilterBlobs( ctx, - &azStorageBlob.ServiceFilterBlobsOptions{ - Where: &tagsFilter, - }, + tagsFilter, + nil, ) if err != nil { - return filteredBlobs, err + return nil, err } - filteredBlobs = result.Blobs + filteredBlobs = result.FilterBlobSegment.Blobs return filteredBlobs, err } @@ -52,11 +54,7 @@ type ListerResponse struct { Marker ListMarker // nil if no more pages Prefix string - // Standard request status things - StatusCode int // For If- header fails, err can be nil and code can be 304 - Status string - - Items []*azStorageBlob.BlobItemInternal + Items []*container.BlobItem } func (azp *Storer) List(ctx context.Context, opts ...Option) (*ListerResponse, error) { @@ -65,38 +63,45 @@ func (azp *Storer) List(ctx context.Context, opts ...Option) (*ListerResponse, e for _, opt := range opts { opt(options) } - o := azStorageBlob.ContainerListBlobsFlatOptions{ + o := azStorageBlob.ListBlobsFlatOptions{ Marker: options.listMarker, + Include: container.ListBlobsInclude{ + Metadata: options.listIncludeMetadata, + Tags: options.listIncludeTags, + }, } if options.listPrefix != "" { o.Prefix = &options.listPrefix } - if options.listIncludeTags { - o.Include = append(o.Include, azStorageBlob.ListBlobsIncludeItemTags) - } - if options.listIncludeMetadata { - o.Include = append(o.Include, azStorageBlob.ListBlobsIncludeItemMetadata) - } // TODO: v1.21 feature which would be great // if options.listDelim != "" { // } - r := &ListerResponse{} - pager := azp.containerClient.ListBlobsFlat(&o) - if !pager.NextPage(ctx) { - return r, nil - } - resp := pager.PageResponse() - r.Status = resp.RawResponse.Status - r.StatusCode = resp.RawResponse.StatusCode + r := &ListerResponse{Items: []*container.BlobItem{}} + // blob listings are returned across multiple pages + pager := azp.containerClient.NewListBlobsFlatPager(&o) + resp, err := pager.NextPage(ctx) + if err != nil { + return nil, err + } if resp.Prefix != nil { r.Prefix = *resp.Prefix } - - // Note: we pass on the azure type otherwise we would be copying for no good - // reason. let the caller decided how to deal with that - r.Items = resp.Segment.BlobItems - + r.Items = append(r.Items, resp.Segment.BlobItems...) + // continue fetching pages until no more remain + for pager.More() { + // advance to the next page + resp, err := pager.NextPage(ctx) + if err != nil { + return nil, err + } + if resp.Prefix != nil { + r.Prefix = *resp.Prefix + } + // Note: we pass on the azure type otherwise we would be copying for no good + // reason. let the caller decided how to deal with that + r.Items = append(r.Items, resp.Segment.BlobItems...) + } return r, nil } diff --git a/azblob/options.go b/azblob/options.go index 012d5cb..6cd1ec6 100644 --- a/azblob/options.go +++ b/azblob/options.go @@ -34,7 +34,7 @@ func (g GetMetadata) String() string { // StorerOptions - optional args for specifying optional behaviour type StorerOptions struct { leaseID string - metadata map[string]string + metadata map[string]*string tags map[string]string getMetadata GetMetadata getTags bool @@ -150,7 +150,7 @@ func WithLeaseID(leaseID string) Option { } // WithMetadata specifies metadata to add - Write() only -func WithMetadata(metadata map[string]string) Option { +func WithMetadata(metadata map[string]*string) Option { return func(a *StorerOptions) { a.metadata = metadata } diff --git a/azblob/put.go b/azblob/put.go index 4a9483d..115914d 100644 --- a/azblob/put.go +++ b/azblob/put.go @@ -6,6 +6,7 @@ import ( "io" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" "github.com/datatrails/go-datatrails-common/logger" ) @@ -54,24 +55,20 @@ func (azp *Storer) putBlob( return nil, err } - blockBlobClient, err := azp.containerClient.NewBlockBlobClient(identity) - if err != nil { - logger.Sugar.Infof("Cannot get block blob client blob: %v", err) - return nil, ErrorFromError(err) - } + blockBlobClient := azp.containerClient.NewBlockBlobClient(identity) r, err := blockBlobClient.Upload( ctx, body, - &azStorageBlob.BlockBlobUploadOptions{ - BlobAccessConditions: &blobAccessConditions, - Metadata: options.metadata, - TagsMap: options.tags, + &blockblob.UploadOptions{ + AccessConditions: &blobAccessConditions, + Metadata: options.metadata, + Tags: options.tags, }, ) if err != nil { logger.Sugar.Infof("Cannot upload blob: %v", err) return nil, ErrorFromError(err) } - return uploadWriteResponse(r), nil + return uploadUploadResponse(r), nil } diff --git a/azblob/reader.go b/azblob/reader.go index 316f94f..c67f035 100644 --- a/azblob/reader.go +++ b/azblob/reader.go @@ -6,6 +6,7 @@ import ( "fmt" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" "github.com/datatrails/go-datatrails-common/logger" ) @@ -59,7 +60,7 @@ func NewReaderNoAuth(accountName string, url string, container string) (Reader, url, container, ) - azp.serviceClient, err = azStorageBlob.NewServiceClientWithNoCredential( + azp.serviceClient, err = service.NewClientWithNoCredential( url, nil, ) @@ -67,11 +68,7 @@ func NewReaderNoAuth(accountName string, url string, container string) (Reader, logger.Sugar.Infof("unable to create serviceclient %s: %v", azp.containerURL, err) return nil, err } - azp.containerClient, err = azp.serviceClient.NewContainerClient(container) - if err != nil { - logger.Sugar.Infof("unable to create containerclient %s: %v", container, err) - return nil, err - } + azp.containerClient = azp.serviceClient.NewContainerClient(container) return azp, nil } diff --git a/azblob/readresponse.go b/azblob/readresponse.go index e95da6d..c0902e0 100644 --- a/azblob/readresponse.go +++ b/azblob/readresponse.go @@ -7,7 +7,11 @@ import ( "strconv" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/datatrails/go-datatrails-common/logger" "github.com/datatrails/go-datatrails-common/scannedstatus" ) @@ -24,29 +28,30 @@ type ReaderResponse struct { ScannedBadReason string ScannedTimestamp string - BlobClient *azStorageBlob.BlobClient + Status string + StatusCode int + + BlobClient *blob.Client // The following are copied as appropriate from the azure sdk response. // See also WriterResponse ETag *string LastModified *time.Time - Metadata map[string]string // x-ms-meta header - StatusCode int // For If- header fails, err can be nil and code can be 304 - Status string - XMsErrorCode string // will be "ConditioNotMet" for If- header predicate fails, even when err is nil + Metadata map[string]*string // x-ms-meta header + XMsErrorCode string // will be "ConditioNotMet" for If- header predicate fails, even when err is nil } func (r *ReaderResponse) ConditionNotMet() bool { - return r.XMsErrorCode == string(azStorageBlob.StorageErrorCodeConditionNotMet) + return r.XMsErrorCode == string(bloberror.ConditionNotMet) } // Ok returns true if the http status was 200 or 201 // This method is provided for use in combination with specific headers like // If-Match and ETags conditions. In thos circumstances we often get err=nil // but no content. -func (r *ReaderResponse) Ok() bool { - return r.StatusCode == 200 || r.StatusCode == 201 -} +//func (r *ReaderResponse) Ok() bool { +// return r.StatusCode == 200 || r.StatusCode == 201 +//} const ( xMsErrorCodeHeader = "X-Ms-Error-Code" @@ -55,21 +60,19 @@ const ( // normaliseReaderResponseErr propagates appropriate err details to the response // this makes it easier to do consistent checking of responses when using ETags // and other conditional header features. -// -// Does nothing unless err can be handled As(azure-sdk.StorageError) func normaliseReaderResponseErr(err error, rr *ReaderResponse) { if err == nil { return } - var terr *azStorageBlob.StorageError + var terr *azcore.ResponseError if !errors.As(err, &terr) { return } if terr.ErrorCode != "" { rr.XMsErrorCode = string(terr.ErrorCode) switch terr.ErrorCode { - case azStorageBlob.StorageErrorCodeConditionNotMet: + case string(bloberror.ConditionNotMet): rr.Status = "304 " + string(terr.ErrorCode) rr.StatusCode = 304 default: @@ -77,49 +80,49 @@ func normaliseReaderResponseErr(err error, rr *ReaderResponse) { } } -// downloadReaderResponse copies accross the azure sdk response values that are +// downloadReaderResponse copies across the azure sdk response values that are // meaningful to our supported api -func downloadReaderResponse(r azStorageBlob.BlobDownloadResponse, rr *ReaderResponse) error { - rr.Status = r.RawResponse.Status - rr.StatusCode = r.RawResponse.StatusCode +func downloadReaderResponse(r blob.DownloadStreamResponse, rr *ReaderResponse) error { + + var err error rr.LastModified = r.LastModified - rr.ETag = r.ETag + //rr.ETag = r.ETag rr.Metadata = r.Metadata - value, ok := r.RawResponse.Header[xMsErrorCodeHeader] - if ok && len(value) > 0 { - rr.XMsErrorCode = value[0] - } - - s, ok := r.RawResponse.Header["Content-Length"] - if !ok { - return nil - } - if len(s) == 0 { - rr.ContentLength = 0 - return nil - } - - var err error - rr.ContentLength, err = strconv.ParseInt(s[0], 10, 64) + //value, ok := r.RawResponse.Header[xMsErrorCodeHeader] + //if ok && len(value) > 0 { + // rr.XMsErrorCode = value[0] + //} + + //s, ok := r.RawResponse.Header["Content-Length"] + //if !ok { + // return nil + //} + //if len(s) == 0 { + // rr.ContentLength = 0 + // return nil + //} + + //var err error + //rr.ContentLength, err = strconv.ParseInt(s[0], 10, 64) return err } // readerResponseMetadata processes and conditions values from the metadata we have specific support for. -func readerResponseMetadata(resp *ReaderResponse, metaData map[string]string) error { - size, parseErr := strconv.ParseInt(metaData[textproto.CanonicalMIMEHeaderKey(SizeKey)], 10, 64) +func readerResponseMetadata(resp *ReaderResponse, metaData map[string]*string) error { + size, parseErr := strconv.ParseInt(*metaData[textproto.CanonicalMIMEHeaderKey(SizeKey)], 10, 64) if parseErr != nil { logger.Sugar.Infof("cannot get size value: %v", parseErr) return parseErr } resp.Size = size - resp.HashValue = metaData[textproto.CanonicalMIMEHeaderKey(HashKey)] - resp.MimeType = metaData[textproto.CanonicalMIMEHeaderKey(MimeKey)] - resp.TimestampAccepted = metaData[textproto.CanonicalMIMEHeaderKey(TimeKey)] - resp.ScannedStatus = scannedstatus.FromString(metaData[textproto.CanonicalMIMEHeaderKey(scannedstatus.Key)]).String() - resp.ScannedBadReason = metaData[textproto.CanonicalMIMEHeaderKey(scannedstatus.BadReason)] - resp.ScannedTimestamp = metaData[textproto.CanonicalMIMEHeaderKey(scannedstatus.Timestamp)] + resp.HashValue = *metaData[textproto.CanonicalMIMEHeaderKey(HashKey)] + resp.MimeType = *metaData[textproto.CanonicalMIMEHeaderKey(MimeKey)] + resp.TimestampAccepted = *metaData[textproto.CanonicalMIMEHeaderKey(TimeKey)] + resp.ScannedStatus = scannedstatus.FromString(*metaData[textproto.CanonicalMIMEHeaderKey(scannedstatus.Key)]).String() + resp.ScannedBadReason = *metaData[textproto.CanonicalMIMEHeaderKey(scannedstatus.BadReason)] + resp.ScannedTimestamp = *metaData[textproto.CanonicalMIMEHeaderKey(scannedstatus.Timestamp)] // Note: it is fine if these are the same instances resp.Metadata = metaData return nil diff --git a/azblob/storer.go b/azblob/storer.go index 51822fd..c2ea63a 100644 --- a/azblob/storer.go +++ b/azblob/storer.go @@ -7,12 +7,14 @@ import ( "fmt" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" "github.com/datatrails/go-datatrails-common/logger" ) // so we dont have to import azure blob package anywhere else -type ContainerClient = azStorageBlob.ContainerClient -type ServiceClient = azStorageBlob.ServiceClient +type ContainerClient = container.Client +type ServiceClient = service.Client type SharedKeyCredential = azStorageBlob.SharedKeyCredential // Storer implements usage of Reader/Writer backed by azblob diff --git a/azblob/storerazurite.go b/azblob/storerazurite.go index 920a0bd..7f44fe7 100644 --- a/azblob/storerazurite.go +++ b/azblob/storerazurite.go @@ -6,6 +6,8 @@ import ( "os" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" "github.com/datatrails/go-datatrails-common/logger" ) @@ -47,12 +49,12 @@ func NewDevConfigFromEnv() DevConfig { } // GetContainerClient returns the underlying container client -func (s *Storer) GetContainerClient() *azStorageBlob.ContainerClient { +func (s *Storer) GetContainerClient() *container.Client { return s.containerClient } // GetServiceClient returns the underlying service client -func (s *Storer) GetServiceClient() *azStorageBlob.ServiceClient { +func (s *Storer) GetServiceClient() *service.Client { return s.serviceClient } diff --git a/azblob/upload.go b/azblob/upload.go index de89dad..be654d3 100644 --- a/azblob/upload.go +++ b/azblob/upload.go @@ -182,7 +182,7 @@ func (azp *Storer) writeStream( return nil, ErrorFromError(err) } - return uploadStreamWriteResponse(r), nil + return uploadWriteResponse(r), nil } func (azp *Storer) streamReader( diff --git a/azblob/writeresponse.go b/azblob/writeresponse.go index 03f0338..e097c3f 100644 --- a/azblob/writeresponse.go +++ b/azblob/writeresponse.go @@ -4,7 +4,9 @@ import ( "errors" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" ) type WriteResponse struct { @@ -35,7 +37,7 @@ func normaliseWriteResponseErr(err error, wr *WriteResponse) { return } - var terr *azStorageBlob.StorageError + var terr *azcore.ResponseError if !errors.As(err, &terr) { return } @@ -50,12 +52,13 @@ func normaliseWriteResponseErr(err error, wr *WriteResponse) { } } -func uploadStreamWriteResponse(r azStorageBlob.BlockBlobCommitBlockListResponse) *WriteResponse { +func uploadWriteResponse(r blockblob.CommitBlockListResponse) *WriteResponse { w := WriteResponse{ ETag: r.ETag, } w.Status = r.RawResponse.Status w.StatusCode = r.RawResponse.StatusCode + w.LastModified = r.LastModified value, ok := r.RawResponse.Header[xMsErrorCodeHeader] if ok && len(value) > 0 { w.XMsErrorCode = value[0] @@ -64,7 +67,7 @@ func uploadStreamWriteResponse(r azStorageBlob.BlockBlobCommitBlockListResponse) return &w } -func uploadWriteResponse(r azStorageBlob.BlockBlobUploadResponse) *WriteResponse { +func uploadUploadResponse(r blockblob.UploadResponse) *WriteResponse { w := WriteResponse{ ETag: r.ETag, } diff --git a/go.mod b/go.mod index 83cd1a4..4f5345a 100644 --- a/go.mod +++ b/go.mod @@ -15,9 +15,9 @@ replace ( require ( github.com/Azure/azure-sdk-for-go v68.0.0+incompatible - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.4.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 github.com/Azure/go-autorest/autorest v0.11.29 github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 github.com/KimMachineGun/automemlimit v0.2.6 @@ -67,7 +67,7 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect @@ -95,9 +95,9 @@ require ( github.com/sirupsen/logrus v1.8.1 // indirect github.com/stretchr/objx v0.5.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/go.sum b/go.sum index 02b78f9..df50755 100644 --- a/go.sum +++ b/go.sum @@ -21,16 +21,18 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0 h1:Yoicul8bnVdQrhDMTHxdEckRGX01XvwXDHUT9zYZ3k0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.4.0 h1:MxbPJrYY81a8xnMml4qICSq1z2WusPw3jSfdIMupnYM= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.4.0/go.mod h1:pXDkeh10bAqElvd+S5Ppncj+DCKvJGXNa8rRT2R7rIw= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 h1:QSdcrd/UFJv6Bp/CfoVf2SrENpFn9P6Yh8yb+xNhYMM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0 h1:fe+kSd9btgTTeHeUlMTyEsjoe6L/zd+Q61iWEMPwHmc= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0/go.mod h1:T7nxmZ9i42Dqy7kwnn8AZYNjqxd4TloKXdIbhosHSqo= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= github.com/Azure/go-amqp v1.0.0 h1:QfCugi1M+4F2JDTRgVnRw7PYXLXZ9hmqk3+9+oJh3OA= github.com/Azure/go-amqp v1.0.0/go.mod h1:+bg0x3ce5+Q3ahCEXnCsGG3ETpDQe3MEVnOuT2ywPwc= @@ -60,8 +62,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c= -github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ConsenSys/quorum v0.0.0-20221208112643-d318a5aa973a h1:bF/fgAYS+vqY1hJuk+C0vfn4G0c0aDd/axZBWslxrpM= @@ -160,8 +162,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= -github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -576,6 +578,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -638,8 +642,10 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -709,6 +715,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=